Ticket #215: ferret64.patch

ruby/test/unit/largefile/tc_sample_large.rb

@@ +1 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class SampleLargeTest < Test::Unit::TestCase
+  include Ferret::Index
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Utils
+  
+  INDEX_DIR = File.dirname(__FILE__) + "/../../temp/largefile"
+  RECORDS = 750
+  RECORD_SIZE = 10e5
+  
+  def setup
+    @index = Index.new(:path => INDEX_DIR, :create_if_missing => true, :key => :id)
+    create_index! if @index.size == 0 or ENV["RELOAD_LARGE_INDEX"]
+  end
+
+  def test_file_index_created
+    assert @index.size == RECORDS, "Index size should be #{RECORDS}, is #{@index.size}"
+  end
+  
+  def test_keys_work
+    @index << {:content => "foo", :id => RECORDS - 4}
+    assert @index.size == RECORDS, "Index size should be #{RECORDS}, is #{@index.size}"
+  end
+  
+  def test_read_file_after_two_gigs
+    assert @index.reader[RECORDS - 5].load.is_a?Hash
+  end
+  
+  def create_index!
+    @@already_built_large_index ||= false
+    return if @@already_built_large_index
+    @@already_built_large_index = true
+    a = "a"
+    RECORDS.times { |i|
+      seq = (a.succ! + " ") * RECORD_SIZE
+      record = {:id => i, :content => seq}
+        @index << record
+        print "i"
+        STDOUT.flush
+    }
+    puts "o"
+    @index.optimize
+  end
+end

ruby/test/unit/ts_largefile.rb

@@ +1 @@
+require File.join(File.dirname(__FILE__), "../test_helper.rb")
+load_test_dir('unit/largefile')

c/include/index.h

@@ -456 +456 @@
 
 typedef struct Offset
 {
-    int start;
-    int end;
+    off_t start;
+    off_t end;
 } Offset;
 
-extern Offset *offset_new(int start, int end);
+extern Offset *offset_new(off_t start, off_t end);
 
 /****************************************************************************
  *
@@ -617 +617 @@
 /* * * LazyDocField * * */
 typedef struct LazyDocFieldData
 {
-    int   start;
+    off_t   start;
     int   length;
     char *text;
 } LazyDocFieldData;

c/src/index.c

@@ -1375 +1375 @@
     lazy_doc = lazy_doc_new(stored_cnt, fdt_in);
 
     for (i = 0; i < stored_cnt; i++) {
-        int start = 0, end, data_cnt;
+        off_t start = 0, end, data_cnt;
         field_num = is_read_vint(fdt_in);
         fi = fr->fis->fields[field_num];
         data_cnt = is_read_vint(fdt_in);
@@ -1449 +1449 @@
         if (store_offsets) {
             int num_positions = tv->offset_cnt = is_read_vint(fdt_in);
             Offset *offsets = tv->offsets = ALLOC_N(Offset, num_positions);
-            int offset = 0;
+            off_t offset = 0;
             for (i = 0; i < num_positions; i++) {
                 offsets[i].start = offset += is_read_vint(fdt_in);
                 offsets[i].end = offset += is_read_vint(fdt_in);
@@ -1683 +1683 @@
         int last_end = 0;
         os_write_vint(fdt_out, offset_count);  /* write shared prefix length */
         for (i = 0; i < offset_count; i++) {
-            int start = offsets[i].start;
-            int end = offsets[i].end;
+            off_t start = offsets[i].start;
+            off_t end = offsets[i].end;
             os_write_vint(fdt_out, start - last_end);
             os_write_vint(fdt_out, end - start);
             last_end = end;
@@ -4799 +4799 @@
  *
  ****************************************************************************/
 
-Offset *offset_new(int start, int end)
+Offset *offset_new(off_t start, off_t end)
 {
     Offset *offset = ALLOC(Offset);
     offset->start = start;
@@ -5204 +5204 @@
     int doc_num = dw->doc_num;
     int i;
     const int df_size = df->size;
-    int start_offset = 0;
+    off_t start_offset = 0;
 
     if (fld_inv->is_tokenized) {
         Token *tk;