From 1de46569e82cdd90e5b11f278796929767e60bdf Mon Sep 17 00:00:00 2001
From: Gea-Suan Lin <gslin@gslin.org>
Date: Fri, 16 Feb 2024 20:59:15 +0800
Subject: [PATCH] Add a simple test case for tokenizer.

---
 internal/tokenizer/tokenize_test.go | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 internal/tokenizer/tokenize_test.go

diff --git a/internal/tokenizer/tokenize_test.go b/internal/tokenizer/tokenize_test.go
new file mode 100644
index 0000000..f6a071b
--- /dev/null
+++ b/internal/tokenizer/tokenize_test.go
@@ -0,0 +1,14 @@
+package tokenizer_test
+
+import (
+	"testing"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/gslin/go-ir-playground/internal/tokenizer"
+)
+
+func testTokenize(t *testing.T) {
+	a := tokenizer.Tokenize("test")
+	assert.Equal(t, len(a), 1)
+	assert.Equal(t, a[0], "test")
+}