diff --git a/analyse/idf.go b/analyse/idf.go index eeaed53..e4250b4 100644 --- a/analyse/idf.go +++ b/analyse/idf.go @@ -10,45 +10,11 @@ import ( ) var ( - stopWords map[string]string idfLoader *IDFLoader ) func init() { idfLoader = NewIDFLoader() - stopWords = map[string]string{ - "the": "the", - "of": "of", - "is": "is", - "and": "and", - "to": "to", - "in": "in", - "that": "that", - "we": "we", - "for": "for", - "an": "an", - "are": "are", - "by": "bye", - "be": "be", - "as": "as", - "on": "on", - "with": "with", - "can": "can", - "if": "of", - "from": "from", - "which": "which", - "you": "you", - "it": "it", - "this": "this", - "then": "then", - "at": "at", - "have": "have", - "all": "all", - "not": "not", - "one": "one", - "has": "has", - "or": "or", - } } type IDFLoader struct { @@ -103,27 +69,3 @@ func SetIdf(idfFilePath string) error { } return idfLoader.NewPath(idfFilePath) } - -func SetStopWords(stopWordsFilePath string) error { - if !filepath.IsAbs(stopWordsFilePath) { - pwd, err := os.Getwd() - if err != nil { - return err - } - stopWordsFilePath = filepath.Clean(filepath.Join(pwd, stopWordsFilePath)) - } - stopWordsFile, err := os.Open(stopWordsFilePath) - if err != nil { - return err - } - scanner := bufio.NewScanner(stopWordsFile) - for scanner.Scan() { - stopWord := scanner.Text() - stopWord = strings.TrimSpace(stopWord) - stopWords[stopWord] = stopWord - } - if err := scanner.Err(); err != nil { - return err - } - return nil -} diff --git a/analyse/stopwords.go b/analyse/stopwords.go new file mode 100644 index 0000000..c6d4d1a --- /dev/null +++ b/analyse/stopwords.go @@ -0,0 +1,70 @@ +package analyse + +import ( + "bufio" + "os" + "path/filepath" + "strings" +) + +var stopWords map[string]string + +func init() { + stopWords = map[string]string{ + "the": "the", + "of": "of", + "is": "is", + "and": "and", + "to": "to", + "in": "in", + "that": "that", + "we": "we", + "for": "for", + "an": "an", + "are": "are", + "by": "bye", + "be": "be", + "as": "as", + "on": "on", + "with": "with", + "can": "can", + "if": "of", + "from": "from", + "which": "which", + "you": "you", + "it": "it", + "this": "this", + "then": "then", + "at": "at", + "have": "have", + "all": "all", + "not": "not", + "one": "one", + "has": "has", + "or": "or", + } +} + +func SetStopWords(stopWordsFilePath string) error { + if !filepath.IsAbs(stopWordsFilePath) { + pwd, err := os.Getwd() + if err != nil { + return err + } + stopWordsFilePath = filepath.Clean(filepath.Join(pwd, stopWordsFilePath)) + } + stopWordsFile, err := os.Open(stopWordsFilePath) + if err != nil { + return err + } + scanner := bufio.NewScanner(stopWordsFile) + for scanner.Scan() { + stopWord := scanner.Text() + stopWord = strings.TrimSpace(stopWord) + stopWords[stopWord] = stopWord + } + if err := scanner.Err(); err != nil { + return err + } + return nil +}