diff --git a/cmd/ir-tfidf/main.go b/cmd/ir-tfidf/main.go index a0ef35c..66c4aae 100644 --- a/cmd/ir-tfidf/main.go +++ b/cmd/ir-tfidf/main.go @@ -4,15 +4,15 @@ import ( "fmt" "github.com/gslin/go-ir-playground/internal/artifact" - "github.com/gslin/go-ir-playground/internal/tokenize" + "github.com/gslin/go-ir-playground/internal/tokenizer" ) func main() { articles := artifact.Read("data/articles.json") for _, article := range articles { - title_bag := tokenize.Tokenize(article.Title) - body_bag := tokenize.Tokenize(article.Body) + title_bag := tokenizer.Tokenize(article.Title) + body_bag := tokenizer.Tokenize(article.Body) fmt.Printf("title_bag = %v\n", title_bag) fmt.Printf("body_bag = %v\n", body_bag) diff --git a/internal/tokenize/tokenize.go b/internal/tokenizer/tokenize.go similarity index 92% rename from internal/tokenize/tokenize.go rename to internal/tokenizer/tokenize.go index e7d2750..c8f783f 100644 --- a/internal/tokenize/tokenize.go +++ b/internal/tokenizer/tokenize.go @@ -1,4 +1,4 @@ -package tokenize +package tokenizer import ( "slices"