Changeset b82c99a2260ab756fe85d7b61da340581dc3b461

Timestamp:

06/14/08 13:24:55 (7 months ago)

Author:

David Balmain <dbalmain@…>

Parents:

0c24555597e26ee38f38d8910d3f7a6612393ce1

Children:

97564968f9aed076ddedd60a97b504b972e3f7fc

git-committer:

David Balmain <dbalmain@gmail.com> / 2008-06-14T13:24:55Z+1000

Message:

Added utf-8 standard analyzer. Back up to speed.

Added utf-8 standard analyzer which is on par with the legacy standard
analyzer. I'm still not happy with this solution though. I think the best idea
right now is to write our own generator which can make use of C's standard
multibyte functions. This shouldn't be too hard as we aren't using many of
Ragel's advanced features anyway.

Location:

c

Files:

• include/analysis.h (modified) (5 diffs)
• include/internal.h (modified) (4 diffs)
• include/scanner.h (modified) (1 diff)
• src/analysis.c (modified) (5 diffs)
• src/scanner.h (modified) (2 diffs)
• src/scanner_utf8.c (added)
• src/scanner_utf8.rl (added)
• src/uchar-utf8.rl (modified) (1 diff)

c/include/analysis.h

@@ -46 +46 @@
 struct FrtTokenStream
 {
-    char        *t;             /* ptr used to scan text */
-    char        *text;
-    FrtToken       *(*next)(FrtTokenStream *ts);
-    FrtTokenStream *(*reset)(FrtTokenStream *ts, char *text);
-    FrtTokenStream *(*clone_i)(FrtTokenStream *ts);
-    void         (*destroy_i)(FrtTokenStream *ts);
-    int          ref_cnt;
+    char            *t;             /* ptr used to scan text */
+    char            *text;
+    FrtToken        *(*next)(FrtTokenStream *ts);
+    FrtTokenStream  *(*reset)(FrtTokenStream *ts, char *text);
+    FrtTokenStream  *(*clone_i)(FrtTokenStream *ts);
+    void            (*destroy_i)(FrtTokenStream *ts);
+    int             ref_cnt;
 };
 
@@ -68 +68 @@
 {
     FrtCachedTokenStream super;
-    mbstate_t         state;
+    mbstate_t            state;
 } FrtMultiByteTokenStream;
 
+typedef enum 
+{
+    FRT_STT_ASCII,
+    FRT_STT_MB,
+    FRT_STT_UTF8
+} FrtStandardTokenizerType;
+
 typedef struct FrtStandardTokenizer
 {
-    FrtCachedTokenStream super;
-    bool is_ascii;
+    FrtCachedTokenStream     super;
+    FrtStandardTokenizerType type;
 } FrtStandardTokenizer;
 
@@ -141 +148 @@
 extern FrtTokenStream *frt_standard_tokenizer_new();
 extern FrtTokenStream *frt_mb_standard_tokenizer_new();
+extern FrtTokenStream *frt_utf8_standard_tokenizer_new();
 
 extern FrtTokenStream *frt_legacy_standard_tokenizer_new();
@@ -212 +220 @@
 extern FrtAnalyzer *frt_standard_analyzer_new(bool lowercase);
 extern FrtAnalyzer *frt_mb_standard_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_utf8_standard_analyzer_new(bool lowercase);
 
 extern FrtAnalyzer *frt_standard_analyzer_new_with_words(
@@ -220 +229 @@
     const char **words, bool lowercase);
 extern FrtAnalyzer *frt_mb_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+extern FrtAnalyzer *frt_utf8_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_utf8_standard_analyzer_new_with_words_len(
     const char **words, int len, bool lowercase);
 

c/include/internal.h

@@ -169 +169 @@
 #define STORE_YES                          FRT_STORE_YES
 #define STRING_FIELD_INDEX_CLASS           FRT_STRING_FIELD_INDEX_CLASS
+#define STT_ASCII                          FRT_STT_ASCII
+#define STT_MB                             FRT_STT_MB
+#define STT_UTF8                           FRT_STT_UTF8
 #define TERM_QUERY                         FRT_TERM_QUERY
 #define TERM_VECTOR_NO                     FRT_TERM_VECTOR_NO
@@ -306 +309 @@
 #define SpanTermQuery           FrtSpanTermQuery
 #define StandardTokenizer       FrtStandardTokenizer
+#define StandardTokenizerType   FrtStandardTokenizerType
 #define State                   FrtState
 #define StemFilter              FrtStemFilter
@@ -911 +915 @@
 #define std_scan                                       frt_std_scan
 #define std_scan_mb                                    frt_std_scan_mb
+#define std_scan_utf8                                  frt_std_scan_utf8
 #define stde_new                                       frt_stde_new
 #define ste_clone                                      frt_ste_clone
@@ -985 +990 @@
 #define u64malloc                                      frt_u64malloc
 #define uchar                                          frt_uchar
+#define utf8_standard_analyzer_new                     frt_utf8_standard_analyzer_new
+#define utf8_standard_analyzer_new_with_words          frt_utf8_standard_analyzer_new_with_words
+#define utf8_standard_analyzer_new_with_words_len      frt_utf8_standard_analyzer_new_with_words_len
+#define utf8_standard_tokenizer_new                    frt_utf8_standard_tokenizer_new
 #define vexit                                          frt_vexit
 #define vstrfmt                                        frt_vstrfmt

c/include/scanner.h

@@ -25 +25 @@
                      int *token_length);
 
+void frt_std_scan_utf8(const char *in,
+                       char *out, size_t out_size,
+                       const char **start, const char **end,
+                       int *token_length);
+
 #ifdef __cplusplus
 } // extern "C"

c/src/analysis.c

@@ -568 +568 @@
     Token *tk = &(CTS(ts)->token);
 
-    if (std_tz->is_ascii) {
-        frt_std_scan(ts->t, tk->text, sizeof(tk->text) - 1,
-                     &start, &end, &len);
-    }
-    else {
-        frt_std_scan_mb(ts->t, tk->text, sizeof(tk->text) - 1,
-                        &start, &end, &len);
+    switch (std_tz->type) {
+        case STT_ASCII:
+            frt_std_scan(ts->t, tk->text, sizeof(tk->text) - 1,
+                         &start, &end, &len);
+            break;
+        case STT_MB:
+            frt_std_scan_mb(ts->t, tk->text, sizeof(tk->text) - 1,
+                            &start, &end, &len);
+            break;
+        case STT_UTF8:
+            frt_std_scan_utf8(ts->t, tk->text, sizeof(tk->text) - 1,
+                              &start, &end, &len);
+            break;
     }
 
@@ -606 +612 @@
 {
     TokenStream *ts = std_ts_new();
-    STDTS(ts)->is_ascii = true;
+    STDTS(ts)->type = STT_ASCII;
     return ts;
 }
@@ -613 +619 @@
 {
     TokenStream *ts = std_ts_new();
-    STDTS(ts)->is_ascii = false;
+    STDTS(ts)->type = STT_MB;
+    return ts;
+}
+
+TokenStream *utf8_standard_tokenizer_new()
+{
+    TokenStream *ts = std_ts_new();
+    STDTS(ts)->type = STT_UTF8;
     return ts;
 }
@@ -1526 +1539 @@
 }
 
+Analyzer *utf8_standard_analyzer_new_with_words_len(const char **words,
+                                                  int len, bool lowercase)
+{
+    TokenStream *ts = utf8_standard_tokenizer_new();
+    if (lowercase) {
+        ts = mb_lowercase_filter_new(ts);
+    }
+    ts = hyphen_filter_new(stop_filter_new_with_words_len(ts, words, len));
+    return analyzer_new(ts, NULL, NULL);
+}
+
+Analyzer *utf8_standard_analyzer_new_with_words(const char **words,
+                                              bool lowercase)
+{
+    TokenStream *ts = utf8_standard_tokenizer_new();
+    if (lowercase) {
+        ts = mb_lowercase_filter_new(ts);
+    }
+    ts = hyphen_filter_new(stop_filter_new_with_words(ts, words));
+    return analyzer_new(ts, NULL, NULL);
+}
+
 Analyzer *standard_analyzer_new(bool lowercase)
 {
@@ -1536 +1571 @@
     return mb_standard_analyzer_new_with_words(FULL_ENGLISH_STOP_WORDS,
                                                lowercase);
+}
+
+Analyzer *utf8_standard_analyzer_new(bool lowercase)
+{
+    return utf8_standard_analyzer_new_with_words(FULL_ENGLISH_STOP_WORDS,
+                                                 lowercase);
 }
 

c/src/scanner.h

@@ -1 +1 @@
 #ifndef FRT_SCANNER_H
 #define FRT_SCANNER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 /*
@@ -21 +25 @@
                      int *token_length);
 
+void frt_std_scan_utf8(const char *in,
+                       char *out, size_t out_size,
+                       const char **start, const char **end,
+                       int *token_length);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
 #endif /* FRT_SCANNER */

c/src/uchar-utf8.rl

@@ -8 +8 @@
 
 %%{
-    machine WChar;
+    machine UTF8;
     ualpha = 
         0x41..0x5A              #L&  [26] LATIN CAPITAL LETTER A..LATIN CAPI...