| 68 | | |
| 69 | | /**************************************************************************** |
| 70 | | * |
| 71 | | * TermVectorsReader |
| 72 | | * |
| 73 | | ****************************************************************************/ |
| 74 | | |
| 75 | | TermVectorsReader *tvr_open(Store *store, |
| 76 | | const char *segment, |
| 77 | | FieldInfos *fis) |
| 78 | | { |
| 79 | | TermVectorsReader *tvr = ALLOC(TermVectorsReader); |
| 80 | | char file_name[SEGMENT_NAME_MAX_LENGTH]; |
| 81 | | |
| 82 | | tvr->fis = fis; |
| 83 | | sprintf(file_name, "%s.tvx", segment); |
| 84 | | tvr->tvx_in = store->open_input(store, file_name); |
| 85 | | tvr->size = is_length(tvr->tvx_in) / 12; |
| 86 | | |
| 87 | | sprintf(file_name, "%s.tvd", segment); |
| 88 | | tvr->tvd_in = store->open_input(store, file_name); |
| 89 | | return tvr; |
| 90 | | } |
| 91 | | |
| 92 | | TermVectorsReader *tvr_clone(TermVectorsReader *orig) |
| 93 | | { |
| 94 | | TermVectorsReader *tvr = ALLOC(TermVectorsReader); |
| 95 | | |
| 96 | | memcpy(tvr, orig, sizeof(TermVectorsReader)); |
| 97 | | tvr->tvx_in = is_clone(orig->tvx_in); |
| 98 | | tvr->tvd_in = is_clone(orig->tvd_in); |
| 99 | | |
| 100 | | return tvr; |
| 101 | | } |
| 102 | | |
| 103 | | void tvr_close(TermVectorsReader *tvr) |
| 104 | | { |
| 105 | | is_close(tvr->tvx_in); |
| 106 | | is_close(tvr->tvd_in); |
| 107 | | free(tvr); |
| 108 | | } |
| 109 | | |
| 110 | | static TermVector *tvr_read_term_vector(TermVectorsReader *tvr, int field_num) |
| 111 | | { |
| 112 | | TermVector *tv = ALLOC_AND_ZERO(TermVector); |
| 113 | | InStream *tvd_in = tvr->tvd_in; |
| 114 | | FieldInfo *fi = tvr->fis->fields[field_num]; |
| 115 | | const int num_terms = is_read_vint(tvd_in); |
| 116 | | |
| 117 | | tv->field_num = field_num; |
| 118 | | tv->field = fi->name; |
| 119 | | |
| 120 | | if (num_terms > 0) { |
| 121 | | int i, j, delta_start, delta_len, total_len, freq; |
| 122 | | int store_positions = fi_store_positions(fi); |
| 123 | | int store_offsets = fi_store_offsets(fi); |
| 124 | | uchar buffer[MAX_WORD_SIZE]; |
| 125 | | TVTerm *term; |
| 126 | | |
| 127 | | tv->term_cnt = num_terms; |
| 128 | | tv->terms = ALLOC_AND_ZERO_N(TVTerm, num_terms); |
| 129 | | |
| 130 | | for (i = 0; i < num_terms; i++) { |
| 131 | | term = &(tv->terms[i]); |
| 132 | | /* read delta encoded term */ |
| 133 | | delta_start = is_read_vint(tvd_in); |
| 134 | | delta_len = is_read_vint(tvd_in); |
| 135 | | total_len = delta_start + delta_len; |
| 136 | | is_read_bytes(tvd_in, buffer + delta_start, delta_len); |
| 137 | | buffer[total_len++] = '\0'; |
| 138 | | term->text = (char *)memcpy(ALLOC_N(char, total_len), buffer, |
| 139 | | total_len); |
| 140 | | |
| 141 | | /* read freq */ |
| 142 | | freq = term->freq = is_read_vint(tvd_in); |
| 143 | | |
| 144 | | /* read positions if necessary */ |
| 145 | | if (store_positions) { |
| 146 | | int *positions = term->positions = ALLOC_N(int, freq); |
| 147 | | int pos = 0; |
| 148 | | for (j = 0; j < freq; j++) { |
| 149 | | positions[j] = pos += is_read_vint(tvd_in); |
| 150 | | } |
| 151 | | } |
| 152 | | |
| 153 | | /* read offsets if necessary */ |
| 154 | | } |
| 155 | | if (store_offsets) { |
| 156 | | int num_positions = tv->offset_cnt = is_read_vint(tvd_in); |
| 157 | | Offset *offsets = tv->offsets = ALLOC_N(Offset, num_positions); |
| 158 | | int offset = 0; |
| 159 | | for (i = 0; i < num_positions; i++) { |
| 160 | | offsets[i].start = offset += is_read_vint(tvd_in); |
| 161 | | offsets[i].end = offset += is_read_vint(tvd_in); |
| 162 | | } |
| 163 | | } |
| 164 | | } |
| 165 | | return tv; |
| 166 | | } |
| 167 | | |
| 168 | | Hash *tvr_get_tv(TermVectorsReader *tvr, int doc_num) |
| 169 | | { |
| 170 | | Hash *term_vectors = h_new_str((free_ft)NULL, (free_ft)&tv_destroy); |
| 171 | | int i; |
| 172 | | InStream *tvx_in = tvr->tvx_in; |
| 173 | | InStream *tvd_in = tvr->tvd_in; |
| 174 | | off_t data_ptr, field_index_ptr; |
| 175 | | int field_cnt; |
| 176 | | int *field_nums; |
| 177 | | |
| 178 | | if (doc_num >= 0 && doc_num < tvr->size) { |
| 179 | | is_seek(tvx_in, 12 * doc_num); |
| 180 | | |
| 181 | | data_ptr = (off_t)is_read_u64(tvx_in); |
| 182 | | field_index_ptr = data_ptr + (off_t)is_read_u32(tvx_in); |
| 183 | | |
| 184 | | /* scan fields to get position of field_num's term vector */ |
| 185 | | is_seek(tvd_in, field_index_ptr); |
| 186 | | |
| 187 | | field_cnt = is_read_vint(tvd_in); |
| 188 | | field_nums = ALLOC_N(int, field_cnt); |
| 189 | | |
| 190 | | for (i = 0; i < field_cnt; i++) { |
| 191 | | field_nums[i] = is_read_vint(tvd_in); |
| 192 | | is_read_vint(tvd_in); /* skip space, we don't need it */ |
| 193 | | } |
| 194 | | is_seek(tvd_in, data_ptr); |
| 195 | | |
| 196 | | for (i = 0; i < field_cnt; i++) { |
| 197 | | TermVector *tv = tvr_read_term_vector(tvr, field_nums[i]); |
| 198 | | h_set(term_vectors, tv->field, tv); |
| 199 | | } |
| 200 | | free(field_nums); |
| 201 | | } |
| 202 | | return term_vectors; |
| 203 | | } |
| 204 | | |
| 205 | | TermVector *tvr_get_field_tv(TermVectorsReader *tvr, |
| 206 | | int doc_num, |
| 207 | | int field_num) |
| 208 | | { |
| 209 | | int i; |
| 210 | | InStream *tvx_in = tvr->tvx_in; |
| 211 | | InStream *tvd_in = tvr->tvd_in; |
| 212 | | off_t data_ptr, field_index_ptr; |
| 213 | | int field_cnt; |
| 214 | | int offset = 0; |
| 215 | | TermVector *tv = NULL; |
| 216 | | |
| 217 | | if (doc_num >= 0 && doc_num < tvr->size) { |
| 218 | | is_seek(tvx_in, 12 * doc_num); |
| 219 | | |
| 220 | | data_ptr = (off_t)is_read_u64(tvx_in); |
| 221 | | field_index_ptr = data_ptr + (off_t)is_read_u32(tvx_in); |
| 222 | | |
| 223 | | /* scan fields to get position of field_num's term vector */ |
| 224 | | is_seek(tvd_in, field_index_ptr); |
| 225 | | |
| 226 | | field_cnt = is_read_vint(tvd_in); |
| 227 | | for (i = 0; i < field_cnt; i++) { |
| 228 | | if ((int)is_read_vint(tvd_in) == field_num) { |
| 229 | | break; |
| 230 | | } |
| 231 | | offset += is_read_vint(tvd_in); /* space taken by field */ |
| 232 | | } |
| 233 | | if (i < field_cnt) { |
| 234 | | /* field was found */ |
| 235 | | is_seek(tvd_in, data_ptr + offset); |
| 236 | | tv = tvr_read_term_vector(tvr, field_num); |
| 237 | | } |
| 238 | | } |
| 239 | | return tv; |
| 240 | | } |
| 241 | | |
| 242 | | /**************************************************************************** |
| 243 | | * |
| 244 | | * TermVectorsWriter |
| 245 | | * |
| 246 | | ****************************************************************************/ |
| 247 | | |
| 248 | | TermVectorsWriter *tvw_open(Store *store, const char *segment, FieldInfos *fis) |
| 249 | | { |
| 250 | | TermVectorsWriter *tvw = ALLOC(TermVectorsWriter); |
| 251 | | char file_name[SEGMENT_NAME_MAX_LENGTH]; |
| 252 | | tvw->fis = fis; |
| 253 | | tvw->fields = ary_new_type_capa(TVField, TV_FIELD_INIT_CAPA); |
| 254 | | |
| 255 | | snprintf(file_name, SEGMENT_NAME_MAX_LENGTH, "%s.tvx", segment); |
| 256 | | tvw->tvx_out = store->new_output(store, file_name); |
| 257 | | |
| 258 | | snprintf(file_name, SEGMENT_NAME_MAX_LENGTH, "%s.tvd", segment); |
| 259 | | tvw->tvd_out = store->new_output(store, file_name); |
| 260 | | |
| 261 | | return tvw; |
| 262 | | } |
| 263 | | |
| 264 | | void tvw_close(TermVectorsWriter *tvw) |
| 265 | | { |
| 266 | | os_close(tvw->tvx_out); |
| 267 | | os_close(tvw->tvd_out); |
| 268 | | ary_free(tvw->fields); |
| 269 | | free(tvw); |
| 270 | | } |
| 271 | | |
| 272 | | void tvw_open_doc(TermVectorsWriter *tvw) |
| 273 | | { |
| 274 | | ary_size(tvw->fields) = 0; |
| 275 | | tvw->tvd_ptr = os_pos(tvw->tvd_out); |
| 276 | | os_write_u64(tvw->tvx_out, tvw->tvd_ptr); |
| 277 | | } |
| 278 | | |
| 279 | | void tvw_close_doc(TermVectorsWriter *tvw) |
| 280 | | { |
| 281 | | int i; |
| 282 | | OutStream *tvd_out = tvw->tvd_out; |
| 283 | | os_write_u32(tvw->tvx_out, (u32)(os_pos(tvw->tvd_out) - tvw->tvd_ptr)); |
| 284 | | os_write_vint(tvd_out, ary_size(tvw->fields)); |
| 285 | | for (i = 0; i < ary_size(tvw->fields); i++) { |
| 286 | | os_write_vint(tvd_out, tvw->fields[i].field_num); |
| 287 | | os_write_vint(tvd_out, tvw->fields[i].size); |
| 288 | | } |
| 289 | | } |
| 290 | | |
| 291 | | void tvw_add_postings(TermVectorsWriter *tvw, |
| 292 | | int field_num, |
| 293 | | PostingList **plists, |
| 294 | | int posting_count, |
| 295 | | Offset *offsets, |
| 296 | | int offset_count) |
| 297 | | { |
| 298 | | int i, delta_start, delta_length; |
| 299 | | const char *last_term = EMPTY_STRING; |
| 300 | | off_t tvd_start_pos = os_pos(tvw->tvd_out); |
| 301 | | OutStream *tvd_out = tvw->tvd_out; |
| 302 | | PostingList *plist; |
| 303 | | Posting *posting; |
| 304 | | Occurence *occ; |
| 305 | | FieldInfo *fi = tvw->fis->fields[field_num]; |
| 306 | | int store_positions = fi_store_positions(fi); |
| 307 | | |
| 308 | | ary_grow(tvw->fields); |
| 309 | | ary_last(tvw->fields).field_num = field_num; |
| 310 | | |
| 311 | | os_write_vint(tvd_out, posting_count); |
| 312 | | for (i = 0; i < posting_count; i++) { |
| 313 | | plist = plists[i]; |
| 314 | | posting = plist->last; |
| 315 | | delta_start = hlp_string_diff(last_term, plist->term); |
| 316 | | delta_length = plist->term_len - delta_start; |
| 317 | | |
| 318 | | os_write_vint(tvd_out, delta_start); /* write shared prefix length */ |
| 319 | | os_write_vint(tvd_out, delta_length); /* write delta length */ |
| 320 | | /* write delta chars */ |
| 321 | | os_write_bytes(tvd_out, |
| 322 | | (uchar *)(plist->term + delta_start), |
| 323 | | delta_length); |
| 324 | | os_write_vint(tvd_out, posting->freq); |
| 325 | | last_term = plist->term; |
| 326 | | |
| 327 | | if (store_positions) { |
| 328 | | /* use delta encoding for positions */ |
| 329 | | int last_pos = 0; |
| 330 | | for (occ = posting->first_occ; occ; occ = occ->next) { |
| 331 | | os_write_vint(tvd_out, occ->pos - last_pos); |
| 332 | | last_pos = occ->pos; |
| 333 | | } |
| 334 | | } |
| 335 | | |
| 336 | | } |
| 337 | | |
| 338 | | if (fi_store_offsets(fi)) { |
| 339 | | /* use delta encoding for offsets */ |
| 340 | | int last_end = 0; |
| 341 | | os_write_vint(tvd_out, offset_count); /* write shared prefix length */ |
| 342 | | for (i = 0; i < offset_count; i++) { |
| 343 | | int start = offsets[i].start; |
| 344 | | int end = offsets[i].end; |
| 345 | | os_write_vint(tvd_out, start - last_end); |
| 346 | | os_write_vint(tvd_out, end - start); |
| 347 | | last_end = end; |
| 348 | | } |
| 349 | | } |
| 350 | | |
| 351 | | ary_last(tvw->fields).size = os_pos(tvd_out) - tvd_start_pos; |
| 352 | | } |
| 353 | | |
| 354 | | |
