Changeset 0c11a534a1b97b7addf59c733a31bcbe1120fcc9

Timestamp:

04/25/08 00:09:25 (9 months ago)

Author:

dave <dave@…>

Parents:

8eb44963185f8afa008b1abfe3f6fad0839b2efd

Children:

a04da26a0a26ed33041c9059b44ece39727fc2a1

git-committer:

dave <dave@06fd6eb0-0002-0410-a719-e5602cce40bc> / 2008-04-24T14:09:25Z+0000

Message:

Changed ruby bindings to use ruby symbols

Ruby bindings now replace Ferret's Symbol implementation with its own. The main
advantage of this is that we don't need to do as much to convert Ferret's data
types into Ruby data types.

An added bonus of this was that it highlighted all the places where the current
Ferret Symbol implementation was broken so it is much more solid now.

git-svn-id: svn+ssh://davebalmain.com/home/dave/repos/ferret/trunk@1043 06fd6eb0-0002-0410-a719-e5602cce40bc

Files:

• c/include/index.h (modified) (2 diffs)
• c/include/internal.h (modified) (2 diffs)
• c/include/store.h (modified) (1 diff)
• c/include/symbol.h (modified) (2 diffs)
• c/src/index.c (modified) (5 diffs)
• c/src/q_multi_term.c (modified) (8 diffs)
• c/src/q_parser.c (modified) (1 diff)
• c/src/q_parser.y (modified) (1 diff)
• c/src/q_phrase.c (modified) (7 diffs)
• c/src/q_range.c (modified) (2 diffs)
• c/src/q_span.c (modified) (1 diff)
• c/src/q_term.c (modified) (2 diffs)
• c/src/search.c (modified) (1 diff)
• c/src/store.c (modified) (1 diff)
• c/test/test_fields.c (modified) (1 diff)
• c/test/test_filter.c (modified) (2 diffs)
• c/test/test_index.c (modified) (4 diffs)
• ruby/.gitignore (modified) (1 diff)
• ruby/Rakefile (modified) (2 diffs)
• ruby/ext/extconf.rb (modified) (1 diff)
• ruby/ext/ferret.c (modified) (2 diffs)
• ruby/ext/ferret.h (modified) (1 diff)
• ruby/ext/r_analysis.c (modified) (2 diffs)
• ruby/ext/r_index.c (modified) (11 diffs)
• ruby/ext/r_search.c (modified) (3 diffs)
• ruby/ext/symbol.c (added)
• ruby/ext/symbol.h (added)

c/include/index.h

@@ -648 +648 @@
 struct FrtLazyDoc
 {
-    FrtHash *field_dict;
+    FrtHash *field_dictionary;
     int size;
     FrtLazyDocField **fields;
@@ -655 +655 @@
 
 extern void frt_lazy_doc_close(FrtLazyDoc *self);
+extern FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, FrtSymbol field);
 
 /****************************************************************************

c/include/internal.h

@@ -74 +74 @@
 #define HS_MIN_SIZE                        FRT_HS_MIN_SIZE
 #define I                                  FRT_I
-#define IF                                 FRT_IF
 #define INDEX_ERROR                        FRT_INDEX_ERROR
 #define INDEX_INTERVAL                     FRT_INDEX_INTERVAL
@@ -645 +644 @@
 #define lazy_df_get_data                        frt_lazy_df_get_data
 #define lazy_doc_close                          frt_lazy_doc_close
+#define lazy_doc_get                            frt_lazy_doc_get
 #define legacy_standard_tokenizer_new           frt_legacy_standard_tokenizer_new
 #define letter_analyzer_new                     frt_letter_analyzer_new

c/include/store.h

@@ -566 +566 @@
 
 /**
+ * Write a string with known length to the FrtOutStream. A string is an
+ * integer +length+ in VINT format (see frt_os_write_vint) followed by
+ * +length+ bytes. The string can then be read using frt_is_read_string.
+ *
+ * @param os FrtOutStream to write to
+ * @param str the string to write
+ * @param len the length of the string to write
+ * @raise FRT_IO_ERROR if there is an error writing to the file-system
+ */
+extern FRT_INLINE void os_write_string_len(FrtOutStream *os,
+                                           const char *str,
+                                           int len);
+
+/**
  * Write a string to the FrtOutStream. A string is an integer +length+ in VINT
  * format (see frt_os_write_vint) followed by +length+ bytes. The string can then

c/include/symbol.h

@@ -1 @@
-#ifndef _SYMBOL_H
-#define _SYMBOL_H
+#ifndef FRT_SYMBOL_H
+#define FRT_SYMBOL_H
 
 typedef struct Frt__Symbol {
@@ -13 +13 @@
 
 #define FRT_I frt_intern
-#define FRT_IF frt_intern_and_free
 #define FRT_S(sym) ((const char *)sym)
 

c/src/index.c

@@ -450 +450 @@
     for (i = 0; i < fis_size; i++) {
         fi = fis->fields[i];
-        os_write_string(os, (char *)fi->name);
+        os_write_string(os, S(fi->name));
         tmp.f = fi->boost;
         os_write_u32(os, tmp.i);
@@ -1470 +1470 @@
 {
     LazyDoc *self = ALLOC(LazyDoc);
-    self->field_dict = h_new_str(NULL, (free_ft)&lazy_df_destroy);
+    self->field_dictionary = h_new_ptr((free_ft)&lazy_df_destroy);
     self->size = size;
     self->fields = ALLOC_AND_ZERO_N(LazyDocField *, size);
@@ -1479 +1479 @@
 void lazy_doc_close(LazyDoc *self)
 {
-    h_destroy(self->field_dict);
+    h_destroy(self->field_dictionary);
     is_close(self->fields_in);
     free(self->fields);
@@ -1488 +1488 @@
 {
     self->fields[i] = lazy_df;
-    h_set(self->field_dict, lazy_df->name, lazy_df);
+    h_set(self->field_dictionary, lazy_df->name, lazy_df);
     lazy_df->doc = self;
+}
+
+LazyDocField *frt_lazy_doc_get(LazyDoc *self, Symbol field)
+{
+    return (LazyDocField *)h_get(self->field_dictionary, field);
 }
 
@@ -1716 +1721 @@
 Hash *fr_get_tv(FieldsReader *fr, int doc_num)
 {
-    Hash *term_vectors = h_new_str((free_ft)NULL, (free_ft)&tv_destroy);
+    Hash *term_vectors = h_new_ptr((free_ft)&tv_destroy);
     int i;
     InStream *fdx_in = fr->fdx_in;

c/src/q_multi_term.c

@@ -261 +261 @@
                 expl_new(sim_tf(self->similarity, (float)freq) * tdew->boost,
                          "tf(term_freq(%s:%s)=%d)^%f",
-                         mtsc->field, tdew->term, freq, tdew->boost));
+                         S(mtsc->field), tdew->term, freq, tdew->boost));
 
             total_score += sim_tf(self->similarity, (float)freq) * tdew->boost;
@@ -385 +385 @@
     char *query_str;
     MultiTermQuery *mtq = MTQ(self->query);
-    Symbol field = mtq->field;
+    const char *field = S(mtq->field);
     PriorityQueue *bt_pq = mtq->boosted_terms;
     int i;
@@ -391 +391 @@
     char *doc_freqs = NULL;
     size_t len = 0, pos = 0;
-    const int field_num = fis_get_field_num(ir->fis, field);
+    const int field_num = fis_get_field_num(ir->fis, mtq->field);
 
     if (field_num < 0) {
         return expl_new(0.0, "field \"%s\" does not exist in the index",
-                        (char *)field);
+                        field);
     }
 
@@ -416 +416 @@
     sprintf(doc_freqs + pos, "= %d", total_doc_freqs);
 
-    idf_expl1 = expl_new(self->idf, "idf(%s:<%s>)", (char *)field, doc_freqs);
-    idf_expl2 = expl_new(self->idf, "idf(%s:<%s>)", (char *)field, doc_freqs);
+    idf_expl1 = expl_new(self->idf, "idf(%s:<%s>)", field, doc_freqs);
+    idf_expl2 = expl_new(self->idf, "idf(%s:<%s>)", field, doc_freqs);
     free(doc_freqs);
 
@@ -455 +455 @@
         : (float)0.0;
     field_norm_expl = expl_new(field_norm, "field_norm(field=%s, doc=%d)",
-                               (char *)field, doc_num);
+                               field, doc_num);
 
     expl_add_detail(field_expl, field_norm_expl);
@@ -503 +503 @@
  ***************************************************************************/
 
-static char *multi_tq_to_s(Query *self, Symbol curr_field)
+static char *multi_tq_to_s(Query *self, Symbol default_field)
 {
     int i;
@@ -509 +509 @@
     BoostedTerm *bt;
     char *buffer, *bptr;
-    int flen = (int)sym_len(MTQ(self)->field);
+    const char *field = S(MTQ(self)->field);
+    int flen = (int)strlen(field);
     int tlen = 0;
 
@@ -519 +520 @@
     bptr = buffer = ALLOC_N(char, tlen + flen + 35);
 
-    if (curr_field != MTQ(self)->field) {
-        bptr += sprintf(bptr, "%s:", S(MTQ(self)->field));
+    if (default_field != MTQ(self)->field) {
+        bptr += sprintf(bptr, "%s:", field);
     }
 

c/src/q_parser.c

@@ -2907 +2907 @@
     /* make sure all_fields contains the default fields */
     self->analyzer = analyzer;
-    self->ts_cache = h_new_str(NULL, (free_ft)&ts_deref);
+    self->ts_cache = h_new_ptr((free_ft)&ts_deref);
     self->buf_index = 0;
     self->dynbuf = NULL;

c/src/q_parser.y

@@ -1170 +1170 @@
     /* make sure all_fields contains the default fields */
     self->analyzer = analyzer;
-    self->ts_cache = h_new_str(NULL, (free_ft)&ts_deref);
+    self->ts_cache = h_new_ptr((free_ft)&ts_deref);
     self->buf_index = 0;
     self->dynbuf = NULL;

c/src/q_phrase.c

@@ -533 +533 @@
     size_t len = 0, pos = 0;
     const int field_num = fis_get_field_num(ir->fis, phq->field);
+    const char *field = S(phq->field);
 
     if (field_num < 0) {
-        return expl_new(0.0, "field \"%s\" does not exist in the index", phq->field);
+        return expl_new(0.0, "field \"%s\" does not exist in the index", field);
     }
 
@@ -564 +565 @@
     doc_freqs[pos] = 0;
 
-    idf_expl1 = expl_new(self->idf, "idf(%s:<%s>)", phq->field, doc_freqs);
-    idf_expl2 = expl_new(self->idf, "idf(%s:<%s>)", phq->field, doc_freqs);
+    idf_expl1 = expl_new(self->idf, "idf(%s:<%s>)", field, doc_freqs);
+    idf_expl2 = expl_new(self->idf, "idf(%s:<%s>)", field, doc_freqs);
     free(doc_freqs);
 
@@ -599 +600 @@
         : (float)0.0;
     field_norm_expl = expl_new(field_norm, "field_norm(field=%s, doc=%d)",
-                               phq->field, doc_num);
+                               field, doc_num);
 
     expl_add_detail(field_expl, field_norm_expl);
@@ -893 +894 @@
     const int pos_cnt = phq->pos_cnt;
     PhrasePosition *positions = phq->positions;
+    const char *field = S(phq->field);
+    int flen = strlen(field);
 
     int i, j, buf_index = 0, pos, last_pos;
@@ -900 +903 @@
     if (phq->pos_cnt == 0) {
         if (default_field != phq->field) {
-            return strfmt("%s:\"\"", phq->field);
+            return strfmt("%s:\"\"", field);
         }
         else {
@@ -910 +913 @@
     qsort(positions, pos_cnt, sizeof(PhrasePosition), &phrase_pos_cmp);
 
-    len = sym_len(phq->field) + 1;
+    len = flen + 1;
 
     for (i = 0; i < pos_cnt; i++) {
@@ -926 +929 @@
 
     if (default_field != phq->field) {
-        len = sym_len(phq->field);
-        memcpy(buffer, phq->field, len);
-        buffer[len] = ':';
-        buf_index += len + 1;
+        memcpy(buffer, field, flen);
+        buffer[flen] = ':';
+        buf_index += flen + 1;
     }
 

c/src/q_range.c

@@ -19 +19 @@
 } Range;
 
-static char *range_to_s(Range *range, Symbol field, float boost)
+static char *range_to_s(Range *range, Symbol default_field, float boost)
 {
     char *buffer, *b;
     size_t flen, llen, ulen;
-
-    flen = sym_len(range->field);
+    const char *field = S(range->field);
+
+    flen = strlen(field);
     llen = range->lower_term ? strlen(range->lower_term) : 0;
     ulen = range->upper_term ? strlen(range->upper_term) : 0;
@@ -30 +31 @@
     b = buffer;
 
-    if (field != range->field) {
-        memcpy(buffer, range->field, flen * sizeof(char));
+    if (default_field != range->field) {
+        memcpy(buffer, field, flen * sizeof(char));
         b += flen;
         *b = ':';

c/src/q_span.c

@@ -1615 +1615 @@
     }
     else {
-        p = strfmt("span_terms(%s:%s)", SpQ(self)->field, terms);
+        p = strfmt("span_terms(%s:%s)", S(SpQ(self)->field), terms);
     }
     free(terms);

c/src/q_term.c

@@ -106 +106 @@
     return expl_new(sim_tf(self->similarity, (float)tf),
                     "tf(term_freq(%s:%s)=%d)",
-                    TQ(query)->field, TQ(query)->term, tf);
+                    S(TQ(query)->field), TQ(query)->term, tf);
 }
 
@@ -266 +266 @@
 static char *tq_to_s(Query *self, Symbol default_field)
 {
-    size_t flen = sym_len(TQ(self)->field);
+    const char *field = S(TQ(self)->field);
+    size_t flen = strlen(field);
     size_t tlen = strlen(TQ(self)->term);
     char *buffer = ALLOC_N(char, 34 + flen + tlen);
     char *b = buffer;
     if (default_field != TQ(self)->field) {
-        memcpy(b, TQ(self)->field, sizeof(char) * flen);
+        memcpy(b, field, sizeof(char) * flen);
         b[flen] = ':';
         b += flen + 1;

c/src/search.c

@@ -848 +848 @@
     LazyDocField *lazy_df = NULL;
     if (lazy_doc) {
-        lazy_df = (LazyDocField *)h_get(lazy_doc->field_dict, field);
+        lazy_df = lazy_doc_get(lazy_doc, field);
     }
     if (tv && lazy_df && tv->term_cnt > 0 && tv->terms[0].positions != NULL

c/src/store.c

@@ -599 +599 @@
 }
 
+INLINE void os_write_string_len(OutStream *os, const char *str, int len)
+{
+    os_write_vint(os, len);
+    os_write_bytes(os, (uchar *)str, len);
+}
 void os_write_string(OutStream *os, const char *str)
 {
-    int len = (int)strlen(str);
-    os_write_vint(os, len);
-
-    os_write_bytes(os, (uchar *)str, len);
+    os_write_string_len(os, str, (int)strlen(str));
 }
 

c/test/test_fields.c

@@ -545 +545 @@
     store_deref(store);
 
-    lazy_df = (LazyDocField *)h_get(lazy_doc->field_dict, "stored");
+    lazy_df = lazy_doc_get(lazy_doc, I("stored"));
     Apnull(lazy_doc->fields[0]->data[0].text);
     Asequal("this is a stored field", text=lazy_df_get_data(lazy_df, 0));

c/test/test_filter.c

@@ -250 +250 @@
     float is_ok = 0.0;
     LazyDoc *lazy_doc = searcher_get_lazy_doc(sea, doc_num);
-    LazyDocField *lazy_df = (LazyDocField *)h_get(lazy_doc->field_dict, "num");
+    LazyDocField *lazy_df = lazy_doc_get(lazy_doc, I("num"));
     char *num = lazy_df_get_data(lazy_df, 0);
     (void)score;
@@ -268 +268 @@
     float distance = 0.0;
     LazyDoc *lazy_doc = searcher_get_lazy_doc(sea, doc_num);
-    LazyDocField *lazy_df = (LazyDocField *)h_get(lazy_doc->field_dict, "num");
+    LazyDocField *lazy_df = lazy_doc_get(lazy_doc, I("num"));
     char *num = lazy_df_get_data(lazy_df, 0);
     (void)score;

c/test/test_index.c

@@ -1736 +1736 @@
     tvs = ir->term_vectors(ir, 3);
     Aiequal(3, tvs->size);
-    tv = (TermVector *)h_get(tvs, "author");
+    tv = (TermVector *)h_get(tvs, I("author"));
     if (Apnotnull(tv)) {
         Asequal("author", tv->field);
@@ -1743 +1743 @@
         Apnull(tv->offsets);
     }
-    tv = (TermVector *)h_get(tvs, "body");
+    tv = (TermVector *)h_get(tvs, I("body"));
     if (Apnotnull(tv)) {
         Asequal("body", tv->field);
         Aiequal(4, tv->term_cnt);
     }
-    tv = (TermVector *)h_get(tvs, "title");
+    tv = (TermVector *)h_get(tvs, I("title"));
     if (Apnotnull(tv)) {
         Asequal("title", tv->field);
@@ -1821 +1821 @@
 
     lz_doc = ir->get_lazy_doc(ir, 0);
-    lz_df1 = (LazyDocField *)h_get(lz_doc->field_dict, changing_field);
-    lz_df2 = (LazyDocField *)h_get(lz_doc->field_dict, compressed_field);
+    lz_df1 = lazy_doc_get(lz_doc, changing_field);
+    lz_df2 = lazy_doc_get(lz_doc, compressed_field);
     Asequal(lazy_df_get_data(lz_df1, 0), lazy_df_get_data(lz_df2, 0));
     lazy_doc_close(lz_doc);
 
     lz_doc = ir->get_lazy_doc(ir, 2);
-    lz_df1 = (LazyDocField *)h_get(lz_doc->field_dict, tag);
-    lz_df2 = (LazyDocField *)h_get(lz_doc->field_dict, compressed_field);
+    lz_df1 = lazy_doc_get(lz_doc, tag);
+    lz_df2 = lazy_doc_get(lz_doc, compressed_field);
     for (i = 0; i < 4; i++) {
         Asequal(lazy_df_get_data(lz_df1, i), lazy_df_get_data(lz_df2, i));
@@ -1835 +1835 @@
 
     lz_doc = ir->get_lazy_doc(ir, 2);
-    lz_df1 = (LazyDocField *)h_get(lz_doc->field_dict, tag);
-    lz_df2 = (LazyDocField *)h_get(lz_doc->field_dict, compressed_field);
+    lz_df1 = lazy_doc_get(lz_doc, tag);
+    lz_df2 = lazy_doc_get(lz_doc, compressed_field);
     lazy_df_get_bytes(lz_df1, buf1, 5, 11);
     lazy_df_get_bytes(lz_df2, buf2, 5, 11);

ruby/.gitignore

@@ -138 +138 @@
 ext/store.c
 ext/store.h
-ext/symbol.c
-ext/symbol.h
 ext/term_vectors.c
 ext/win32.h

ruby/Rakefile

@@ -55 +55 @@
                      "../c/lib/libstemmer_c/include/libstemmer.[h]"]
   EXT_SRC.exclude('../c/**/ind.[ch]',
+                  '../c/**/symbol.[ch]',
                   '../c/include/threading.h',
+                  '../c/include/scanner.h',
                   '../c/include/internal.h',
                   '../c/src/lang.c',
@@ -82 +84 @@
     # prepend lib files to avoid conflicts
     file dest_fn => fn do |t|
-      safe_ln fn, dest_fn
+      ln fn, dest_fn
 
       if fn =~ /stemmer/

ruby/ext/extconf.rb

@@ -6 +6 @@
 elsif ENV['FERRET_DEV']
   require 'mkmf'
-  $CFLAGS += " -g -Wall -fno-stack-protector -fno-common -D_FILE_OFFSET_BITS=64 -D_XOPEN_SOURCE=500"
+  $CFLAGS = " -g -Wall -fno-stack-protector -fno-common -D_FILE_OFFSET_BITS=64 -D_XOPEN_SOURCE=500"
+  puts $CFLAGS
   create_makefile("ferret_ext")
 else

ruby/ext/ferret.c

@@ -191 +191 @@
     switch (TYPE(rfield)) {
         case T_SYMBOL:
-            return I(rb_id2name(SYM2ID(rfield)));
+            return SYM2FSYM(rfield);
         case T_STRING:
             return I(rs2s(rfield));
@@ -293 +293 @@
 {
     return rb_struct_new(cTerm,
-                         SYM2RSYM(field),
+                         FSYM2SYM(field),
                          rb_str_new2(text),
                          NULL);