/[zanavi_public1]/navit/navit/maptool/osm_s_index.h
ZANavi

Contents of /navit/navit/maptool/osm_s_index.h

Parent Directory Parent Directory | Revision Log Revision Log


Revision 57 - (show annotations) (download)
Sun Mar 19 16:46:08 2017 UTC (7 years ago) by zoff99
File MIME type: text/plain
File size: 35575 byte(s)
updates
1 /**
2 * ZANavi, Zoff Android Navigation system.
3 * Copyright (C) 2015 Zoff <zoff@zoff.cc>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * version 2 as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20
21
22 #ifdef MAPTOOL_USE_STRINDEX_COMPRESSION
23
24
25
26
27 typedef unsigned char uint8;
28 typedef unsigned short uint16;
29 typedef unsigned int uint;
30
31 #define MZ_MIN(a,b) (((a)<(b))?(a):(b))
32 #define my_min(a,b) (((a) < (b)) ? (a) : (b))
33
34
35
36
37 // -------------------------------
38 // -------------------------------
39
40 int IN_BUF_SIZE;
41 static uint8* s_inbuf;
42 int OUT_BUF_SIZE;
43 static uint8* s_outbuf;
44 int COMP_OUT_BUF_SIZE;
45
46 //** int s_IN_BUF_SIZE = sizeof(struct streets_index_data_block) * 1024;
47 //** int t_IN_BUF_SIZE = sizeof(struct town_index_data_block) * 1024;
48 //** static uint8 s_s_inbuf[(sizeof(struct streets_index_data_block) * 1024)];
49 //** static uint8 t_s_inbuf[(sizeof(struct town_index_data_block) * 1024)];
50
51 //** int s_OUT_BUF_SIZE = sizeof(struct streets_index_data_block) * 1024;
52 //** int t_OUT_BUF_SIZE = sizeof(struct town_index_data_block) * 1024;
53 //** static uint8 s_s_outbuf[(sizeof(struct streets_index_data_block) * 1024)];
54 //** static uint8 t_s_outbuf[(sizeof(struct town_index_data_block) * 1024)];
55
56 int s_COMP_OUT_BUF_SIZE;
57 int t_COMP_OUT_BUF_SIZE;
58
59
60 /* ------ new ------ */
61 int s_IN_BUF_SIZE = sizeof(struct streets_index_data_block) * 2048;
62 int t_IN_BUF_SIZE = sizeof(struct town_index_data_block) * 2048;
63 uint8 *s_s_inbuf;
64 uint8 *t_s_inbuf;
65
66 int s_OUT_BUF_SIZE = (sizeof(struct streets_index_data_block) * 512);
67 int t_OUT_BUF_SIZE = (sizeof(struct town_index_data_block) * 512);
68 uint8 *s_s_outbuf;
69 uint8 *t_s_outbuf;
70 /* ------ new ------ */
71
72
73 // IN_BUF_SIZE is the size of the file read buffer.
74 // IN_BUF_SIZE must be >= 1
75 //**#define IN_BUF_SIZE (1024*512)
76 //**static uint8 s_inbuf[IN_BUF_SIZE];
77
78 // COMP_OUT_BUF_SIZE is the size of the output buffer used during compression.
79 // COMP_OUT_BUF_SIZE must be >= 1 and <= OUT_BUF_SIZE
80 //**#define COMP_OUT_BUF_SIZE (1024*512)
81
82 // OUT_BUF_SIZE is the size of the output buffer used during decompression.
83 // OUT_BUF_SIZE must be a power of 2 >= TINFL_LZ_DICT_SIZE (because the low-level decompressor not only writes, but reads from the output buffer as it decompresses)
84 //#define OUT_BUF_SIZE (TINFL_LZ_DICT_SIZE)
85 //*#define OUT_BUF_SIZE (1024*512)
86 //*static uint8 s_outbuf[OUT_BUF_SIZE];
87
88 // -------------------------------
89 // -------------------------------
90
91
92
93
94 // compression level
95 int street_index_compress_level = 9; // = MZ_BEST_COMPRESSION
96
97 long long compress_file(char *in, char *out, int keep_tempfile)
98 {
99 long long out_size = 0;
100 // compress structure
101 tdefl_compressor *g_deflator;
102 g_deflator = g_new0(tdefl_compressor, 1);
103 // compress structure
104
105 // ok now compress the block (file) -------------------------------
106 const void *next_in = s_inbuf;
107 size_t avail_in = 0;
108 void *next_out = s_outbuf;
109 size_t avail_out = OUT_BUF_SIZE;
110 size_t total_in = 0, total_out = 0;
111
112 FILE *in_uncompr = tempfile("", in, 0);
113 FILE *out_compr = NULL;
114 out_compr = tempfile("", out, 1);
115
116 // Determine input file's size.
117 fseek(in_uncompr, 0, SEEK_END);
118 long file_loc = ftell(in_uncompr);
119 fseek(in_uncompr, 0, SEEK_SET);
120 uint infile_size = (uint)file_loc;
121
122 // The number of dictionary probes to use at each compression level (0-10). 0=implies fastest/minimal possible probing.
123 static const mz_uint s_tdefl_num_probes[11] =
124 { 0, 1, 6, 32, 16, 32, 128, 256, 512, 768, 1500};
125
126 tdefl_status status;
127 uint infile_remaining = infile_size;
128
129 // create tdefl() compatible flags (we have to compose the low-level flags ourselves, or use tdefl_create_comp_flags_from_zip_params() but that means MINIZ_NO_ZLIB_APIS can't be defined).
130 mz_uint comp_flags = TDEFL_WRITE_ZLIB_HEADER | s_tdefl_num_probes[MZ_MIN(10, street_index_compress_level)] | ((street_index_compress_level <= 3) ? TDEFL_GREEDY_PARSING_FLAG : 0);
131 if (!street_index_compress_level)
132 {
133 comp_flags |= TDEFL_FORCE_ALL_RAW_BLOCKS;
134 }
135
136 // Initialize the low-level compressor.
137 status = tdefl_init(g_deflator, NULL, NULL, comp_flags);
138 if (status != TDEFL_STATUS_OKAY)
139 {
140 fprintf(stderr, "tdefl_init() failed!\n");
141 g_free(g_deflator);
142 return 0;
143 }
144 avail_out = COMP_OUT_BUF_SIZE;
145
146 // Compression.
147 for (;; )
148 {
149 size_t in_bytes, out_bytes;
150
151 if (!avail_in)
152 {
153 // Input buffer is empty, so read more bytes from input file.
154 uint n = my_min(IN_BUF_SIZE, infile_remaining);
155
156 if (fread(s_inbuf, 1, n, in_uncompr) != n)
157 {
158 fprintf(stderr, "Failed reading from input file!\n");
159 g_free(g_deflator);
160 return 0;
161 }
162
163 next_in = s_inbuf;
164 avail_in = n;
165
166 infile_remaining -= n;
167 //printf("Input bytes remaining: %u\n", infile_remaining);
168 }
169
170 in_bytes = avail_in;
171 out_bytes = avail_out;
172 // Compress as much of the input as possible (or all of it) to the output buffer.
173 status = tdefl_compress(g_deflator, next_in, &in_bytes, next_out, &out_bytes, infile_remaining ? TDEFL_NO_FLUSH : TDEFL_FINISH);
174
175 next_in = (const char *)next_in + in_bytes;
176 avail_in -= in_bytes;
177 total_in += in_bytes;
178
179 next_out = (char *)next_out + out_bytes;
180 avail_out -= out_bytes;
181 total_out += out_bytes;
182
183 if ((status != TDEFL_STATUS_OKAY) || (!avail_out))
184 {
185 // Output buffer is full, or compression is done or failed, so write buffer to output file.
186 uint n = COMP_OUT_BUF_SIZE - (uint)avail_out;
187 if (fwrite(s_outbuf, 1, n, out_compr) != n)
188 {
189 fprintf(stderr, "Failed writing to output file!\n");
190 g_free(g_deflator);
191 return 0;
192 }
193 next_out = s_outbuf;
194 avail_out = COMP_OUT_BUF_SIZE;
195 }
196
197 if (status == TDEFL_STATUS_DONE)
198 {
199 // Compression completed successfully.
200 break;
201 }
202 else if (status != TDEFL_STATUS_OKAY)
203 {
204 // Compression somehow failed.
205 fprintf(stderr, "tdefl_compress() failed with status %i!\n", status);
206 g_free(g_deflator);
207 return 0;
208 }
209 }
210
211 fprintf_(stderr, "Total input bytes: %u\n", (mz_uint32)total_in);
212 fprintf_(stderr, "Total output bytes: %u\n", (mz_uint32)total_out);
213
214 out_size = (long long)total_out;
215
216 fclose(in_uncompr);
217 fclose(out_compr);
218
219 if (keep_tempfile == 1)
220 {
221 char *in2;
222 in2 = g_strdup_printf("%s.tmptmp", in);
223 tempfile_rename("", in, in2);
224 g_free(in2);
225 tempfile_rename("", out, in);
226 }
227 else
228 {
229 tempfile_unlink("", in);
230 tempfile_rename("", out, in);
231 }
232
233 g_free(g_deflator);
234
235 return out_size;
236 }
237 #endif
238
239
240 void generate_combined_index_file(FILE *towni, FILE *streeti, FILE *out)
241 {
242 #ifdef MAPTOOL_USE_SQL
243 fseek(towni, 0, SEEK_END);
244 fseek(streeti, 0, SEEK_END);
245 long long towni_size = (long long)ftello(towni);
246 long long streeti_size = (long long)ftello(streeti);
247 fprintf(stderr, "ftell towns=%lld\n", towni_size);
248 fprintf(stderr, "ftell streets=%lld\n", streeti_size);
249
250 fseek(towni, 0, SEEK_SET);
251 fseek(streeti, 0, SEEK_SET);
252
253 // size
254 fwrite(&streeti_size, sizeof(long long), 1, out);
255 // append street index file
256 filecopy(streeti, out);
257 // append town index file
258 filecopy(towni, out);
259
260 #endif
261 }
262
263 char* get_town_name_recursive(long long border_id, int level, GList* man_borders, int country_id_of_town, const char *town_name)
264 {
265 char *ret = NULL;
266 char *ret2 = NULL;
267 char *str2 = NULL;
268 int rc = 0;
269 int admin_level;
270 long long parent_rel_id = 0;
271 long long my_rel_id = 0;
272 struct coord c;
273 struct country_table *result = NULL;
274 double lat, lon;
275
276 // fprintf(stderr, "get town name:bid:%lld level:%d country_id:%d\n", border_id, level, country_id_of_town);
277
278 if ((border_id == 0) && (town_name))
279 {
280 if (country_id_of_town != 999)
281 {
282 result = osm_process_item_by_country_id(country_id_of_town);
283 }
284
285 if (result)
286 {
287 // fprintf(stderr,"== town recursive by country id 1 == country_id:%d country_name:%s townname:%s ==\n", result->countryid, result->names, town_name);
288 if (result->names)
289 {
290 if (strcmp(town_name, result->names))
291 {
292 ret2 = g_strdup_printf("%s, %s", town_name, result->names);
293 ret = ret2;
294 }
295 else
296 {
297 ret2 = g_strdup_printf("%s", town_name);
298 ret = ret2;
299 }
300 return ret;
301 }
302 }
303 }
304 else
305 {
306
307 // select this boundary
308 sqlite3_bind_int64(stmt_bd_005, 1, border_id);
309 rc = sqlite3_step(stmt_bd_005);
310 switch (rc)
311 {
312 case SQLITE_DONE:
313 break;
314 case SQLITE_ROW:
315 // rel_id, parent_rel_id, name
316 my_rel_id = sqlite3_column_int64(stmt_bd_005, 0);
317 parent_rel_id = sqlite3_column_int64(stmt_bd_005, 1);
318 if (my_rel_id < -1)
319 {
320 // manual country border names are excluded (--> shitty crappy hack fix)
321 ret = NULL;
322 }
323 else
324 {
325 ret = g_strdup_printf("%s", sqlite3_column_text(stmt_bd_005, 2));
326 }
327 admin_level = sqlite3_column_int(stmt_bd_005, 3);
328 lat = sqlite3_column_double(stmt_bd_005, 4);
329 lon = sqlite3_column_double(stmt_bd_005, 5);
330 sqlite3_reset(stmt_bd_005);
331
332 // fprintf(stderr, "get town name:parentrel_id=%lld my_rel_id=%lld ret=%s admin_level=%d\n", parent_rel_id, my_rel_id, ret, admin_level);
333
334 if (level < 12)
335 {
336 // fprintf(stderr, "call get_town_name_recursive townname:%s\n", town_name);
337 str2 = get_town_name_recursive(parent_rel_id, (level + 1), man_borders, country_id_of_town, town_name);
338 // fprintf(stderr, "ret2=%s %p\n", str2, str2);
339 if (str2)
340 {
341 if ((ret) && (strcmp(ret, "(null)")) && (strcmp(ret, str2)))
342 {
343 // fprintf(stderr, "ret=%s str2=%s\n", ret, str2);
344 // only non-null townname strings
345 ret2 = g_strdup_printf("%s, %s", ret, str2);
346 }
347 else
348 {
349 // fprintf(stderr, "str2=%s\n", str2);
350 ret2 = g_strdup_printf("%s", str2);
351 }
352 g_free(ret);
353 g_free(str2);
354 ret = ret2;
355 }
356 else if (admin_level > 2)
357 {
358 if ((lat != 999) && (lon != 999))
359 {
360 if (country_id_of_town != 999)
361 {
362 result = osm_process_item_by_country_id(country_id_of_town);
363 }
364
365 if (result)
366 {
367 // fprintf(stderr,"== town recursive by country id 2 == country_id:%d country_name:%s townname:%s ==\n", result->countryid, result->names, ret);
368 if (result->names)
369 {
370 if ((ret) && (strcmp(ret, "(null)")))
371 {
372 // only non-null townname strings
373 ret2 = g_strdup_printf("%s, %s", ret, result->names);
374 }
375 else
376 {
377 ret2 = g_strdup_printf("%s", result->names);
378 }
379 g_free(ret);
380 ret = ret2;
381 }
382 }
383 else
384 {
385 // recursion has ended before admin_level 2 --> try to connect to manual country border
386 c.x = transform_from_geo_lon(lon);
387 c.y = transform_from_geo_lat(lat);
388 result = osm_process_town_by_manual_country_borders(man_borders, &c);
389 if (result)
390 {
391 // fprintf(stderr,"== town recursive by manual_country_borders == country_id:%d country_name:%s townname:%s ==\n", result->countryid, result->names, ret);
392 if (result->names)
393 {
394 if ((ret) && (strcmp(ret, "(null)")))
395 {
396 // only non-null townname strings
397 ret2 = g_strdup_printf("%s, %s", ret, result->names);
398 }
399 else
400 {
401 ret2 = g_strdup_printf("%s", result->names);
402 }
403 g_free(ret);
404 ret = ret2;
405 }
406 }
407 }
408 }
409 }
410 }
411
412 // check again for (null) string
413 //if ((!ret)||(!strcmp(ret, "(null)")))
414 //{
415 // ret2 = g_strdup(" ");
416 // g_free(ret);
417 // ret = ret2;
418 //}
419
420 break;
421 default:
422 fprintf(stderr, "SQL Error: %d\n", rc);
423 break;
424 }
425
426 if (level == 0)
427 {
428 sqlite3_reset(stmt_bd_005);
429 }
430
431 }
432
433 // fprintf(stderr, "return level=%d border_id=%lld ret=%s p.ret=%p\n", level, border_id, ret, ret);
434 return ret;
435 }
436
437 void generate_town_index_file(FILE *out, GList *man_borders)
438 {
439 #ifdef MAPTOOL_USE_SQL
440
441 struct town_index_data_block db;
442 struct town_index_index_block_start is;
443 struct town_index_index_block ib;
444 char *townname = NULL;
445 char *townname2 = NULL;
446 long long town_index_pos1 = 0;
447 long long town_index_pos2 = 0;
448 long long last_len = 0;
449 long long border_id = 0;
450 int town_count = 0;
451 int index_blocks;
452 int index_block_towns;
453 int i;
454 int first;
455 int rc = 0;
456 int current_index_block;
457 int current_index_block_old;
458 char tmp_letter[TOWN_INDEX_TOWN_NAME_SIZE];
459 char *newfilename = NULL;
460 char *newfilename_compr = NULL;
461
462 int chunkSize;
463 int stringLength;
464 int ii22;
465
466 FILE *town_index_index = NULL;
467 FILE *town_index_index_data_block = NULL;
468
469 struct coord c;
470 double lat;
471 double lon;
472
473 // init compression for towns -------------------
474 // init compression for towns -------------------
475
476 /* s_s_outbuf = g_malloc(s_OUT_BUF_SIZE); */
477 t_s_outbuf = g_malloc0(t_OUT_BUF_SIZE);
478 /* s_s_inbuf = g_malloc(s_IN_BUF_SIZE); */
479 t_s_inbuf = g_malloc0(t_IN_BUF_SIZE);
480
481 IN_BUF_SIZE = t_IN_BUF_SIZE;
482 s_inbuf = t_s_inbuf;
483 OUT_BUF_SIZE = t_OUT_BUF_SIZE;
484 s_outbuf = t_s_outbuf;
485 COMP_OUT_BUF_SIZE = t_OUT_BUF_SIZE;
486
487 // init compression for towns -------------------
488 // init compression for towns -------------------
489
490
491
492
493 purge_unused_towns();
494
495
496 sqlite3_step(stmt_town_sel006);
497 town_count = sqlite3_column_int(stmt_town_sel006, 0);
498 sqlite3_reset(stmt_town_sel006);
499
500
501 // calculate number of index blocks ------------------------------------------
502 // calculate number of index blocks ------------------------------------------
503 //
504 index_blocks = 2;
505 //
506 #if 0
507 if (town_count > 1000000)
508 {
509 index_blocks = 1000; // investigate why this cant be higher?? towns will be broken if it is higher ???????? ****** ???????
510 }
511 else if (town_count > 100000)
512 {
513 index_blocks = 100; // investigate why this cant be higher?? towns will be broken if it is higher ???????? ****** ???????
514 }
515 else if (town_count > 10000)
516 {
517 index_blocks = 50; // investigate why this cant be higher?? towns will be broken if it is higher ???????? ****** ???????
518 }
519 else if (town_count > 2000)
520 {
521 index_blocks = 20;
522 }
523 #endif
524
525 #if 1
526 index_blocks = (town_count / 100); // only 100 towns per block
527 if (index_blocks == 0)
528 {
529 index_blocks = 1;
530 }
531
532 if ((index_blocks * 100) < town_count)
533 {
534 index_blocks++;
535 }
536
537 if (index_blocks > 4000)
538 {
539 index_blocks = 4000;
540 }
541 #endif
542 //
543 // calculate number of index blocks ------------------------------------------
544 // calculate number of index blocks ------------------------------------------
545
546
547 is.count_of_index_blocks = index_blocks;
548 ib.offset = sizeof(is.count_of_index_blocks) + is.count_of_index_blocks * sizeof(struct town_index_index_block); // start offset = index size
549 town_index_index = tempfile("", "town_index_index", 1);
550
551 index_block_towns = (town_count / index_blocks);
552 if ((index_block_towns * index_blocks) < town_count)
553 {
554 index_block_towns++;
555 }
556
557 fprintf_(stderr, "towns per block=%d\n", index_block_towns);
558
559 fprintf_(stderr, "index size=%d\n", ib.offset);
560
561 fprintf_(stderr, "ftell=%d\n", ftell(town_index_index));
562 // remember pos1
563 town_index_pos1 = ftello(town_index_index);
564 fwrite(&is, sizeof(struct town_index_index_block_start), 1, town_index_index);
565 // remember pos2
566 town_index_pos1 = ftello(town_index_index);
567 fprintf_(stderr, "ftell=%d\n", ftell(town_index_index));
568
569 current_index_block = 1;
570 current_index_block_old = 0;
571 ib.len = 0;
572 i = -1;
573 first = 1;
574
575 // loop thru all the towns
576 do
577 {
578 rc = sqlite3_step(stmt_town_sel005);
579 switch (rc)
580 {
581 case SQLITE_DONE:
582 break;
583
584 case SQLITE_ROW:
585 i++;
586 townname2 = NULL;
587 db.town_id = sqlite3_column_int64(stmt_town_sel005, 0);
588 db.country_id = sqlite3_column_int(stmt_town_sel005, 1);
589 border_id = sqlite3_column_int64(stmt_town_sel005, 3);
590
591 //fprintf(stderr, "indextown_0:townid=%lld country_id=%d borderid=%lld townname=%s\n", db.town_id, db.country_id, border_id, sqlite3_column_text(stmt_town_sel005, 2));
592
593 if (border_id != -1)
594 {
595 townname = get_town_name_recursive(border_id, 0, man_borders, db.country_id, sqlite3_column_text(stmt_town_sel005, 2));
596 //fprintf(stderr, "indextown_1:%s pnt:%p\n", townname, (int)townname);
597 }
598 else
599 {
600 townname = NULL;
601 }
602
603
604 if (townname == NULL)
605 {
606 //fprintf(stderr, "indextown_2.a\n");
607 // ok see if we can assign a manual border to this town
608 if ((db.town_id != 0) && (db.town_id != -1))
609 {
610 // fprintf(stderr, "indextown_2.a.1\n");
611 if (db.country_id != 999)
612 {
613 // fprintf(stderr, "indextown_2.a.2\n");
614 // try to find a manual country dummy-border-id
615 border_id = osm_process_town_by_manual_country_id(man_borders, db.country_id);
616 if (border_id != -1)
617 {
618 townname = get_town_name_recursive(border_id, 0, man_borders, db.country_id, sqlite3_column_text(stmt_town_sel005, 2));
619 if (townname)
620 {
621 if (strcmp(townname,sqlite3_column_text(stmt_town_sel005, 2)))
622 {
623 townname2 = g_strdup_printf("%s, %s", sqlite3_column_text(stmt_town_sel005, 2), townname);
624 g_free(townname);
625 townname = townname2;
626 townname2 = NULL;
627 // fprintf(stderr, "indextown_2.1a:%s\n", townname);
628 }
629 else
630 {
631 townname2 = g_strdup_printf("%s", townname);
632 g_free(townname);
633 townname = townname2;
634 townname2 = NULL;
635 // fprintf(stderr, "indextown_2.1b:%s\n", townname);
636 }
637 }
638 }
639 }
640 else
641 {
642 // fprintf(stderr, "indextown_2.a.7\n");
643
644 // try to find a manual country dummy-border-id
645 lat = sqlite3_column_double(stmt_town_sel005, 4);
646 lon = sqlite3_column_double(stmt_town_sel005, 5);
647 c.x = transform_from_geo_lon(lon);
648 c.y = transform_from_geo_lat(lat);
649
650 border_id = osm_process_street_by_manual_country_borders(man_borders, &c);
651 if (border_id != -1)
652 {
653 // fprintf(stderr, "indextown_2.a.8:bid=%lld\n", border_id);
654
655 townname = get_town_name_recursive(border_id, 0, man_borders, db.country_id, sqlite3_column_text(stmt_town_sel005, 2));
656 if (townname)
657 {
658 // fprintf(stderr, "indextown_2.a.9\n");
659
660 townname2 = g_strdup_printf("%s, %s", sqlite3_column_text(stmt_town_sel005, 2), townname);
661 g_free(townname);
662 townname = townname2;
663 townname2 = NULL;
664 // fprintf(stderr, "indextown_2.2:%s\n", townname);
665 }
666 }
667 }
668 }
669 }
670
671 if (townname == NULL)
672 {
673 townname2 = g_strdup_printf("%s", sqlite3_column_text(stmt_town_sel005, 2));
674 //fprintf(stderr, "indextown_3:%s\n", townname2);
675 }
676 else
677 {
678 townname2 = townname;
679 // townname2 = g_strdup_printf("%s", townname); // dont use the column result here, or string will be double!
680 // g_free(townname);
681 }
682
683 fprintf_(stderr, "indextown99:%s\n", townname2);
684 fprintf_(stderr, "i=%d\n", i);
685 fprintf_(stderr, "block =%d\n", current_index_block);
686 fprintf_(stderr, "block old=%d\n", current_index_block_old);
687
688 if ((i + 1) > index_block_towns)
689 {
690 // start new index data block
691 i = 0;
692 current_index_block++;
693 fprintf_(stderr, "incr block=%d\n", current_index_block);
694 }
695
696 if (current_index_block != current_index_block_old)
697 {
698
699 if (first != 1)
700 {
701 // close old datafile
702 fclose(town_index_index_data_block);
703 town_index_index_data_block = NULL;
704
705 if (USE_STREET_INDEX_COMPRESSION == 1)
706 {
707 #ifdef MAPTOOL_USE_STRINDEX_COMPRESSION
708 ib.len = compress_file(newfilename, newfilename_compr, global_keep_tmpfiles);
709 #endif
710 }
711
712 // append to indexfile
713 fprintf_(stderr, "first_id=%lld offset=%lld len=%lld\n", ib.first_id, ib.offset, ib.len);
714 fwrite(&ib, sizeof(struct town_index_index_block), 1, town_index_index);
715
716 if (newfilename)
717 {
718 g_free(newfilename);
719 newfilename = NULL;
720 }
721
722 if (newfilename_compr)
723 {
724 g_free(newfilename_compr);
725 newfilename_compr = NULL;
726 }
727 }
728
729 current_index_block_old = current_index_block;
730 ib.first_id = db.town_id;
731 ib.offset = ib.offset + ib.len;
732
733 // open new datafile
734 newfilename = g_strdup_printf("town_index_index_%d", current_index_block);
735 fprintf_(stderr, "new data file: %s first_id=%lld\n", newfilename, ib.first_id);
736 newfilename_compr = g_strdup_printf("town_index_index_compr_%d", current_index_block);
737 town_index_index_data_block = tempfile("", newfilename, 1);
738 fprintf_(stderr, "town index file %d\n", current_index_block);
739
740 ib.len = 0;
741 }
742
743
744 // now check if we need to split the string into parts
745 if ((strlen(townname2) + 1) > TOWN_INDEX_TOWN_NAME_SIZE)
746 {
747 fprintf_(stderr, " block-split: START\n");
748 chunkSize = TOWN_INDEX_TOWN_NAME_SIZE - 1;
749 stringLength = strlen(townname2);
750 for (ii22 = 0; ii22 < stringLength ; ii22 += chunkSize)
751 {
752 if (ii22 + chunkSize > stringLength)
753 {
754 chunkSize = stringLength - ii22;
755 db.town_name[chunkSize] = '\0'; // make sure string is terminated
756 }
757 strncpy(&db.town_name, (townname2 + ii22), chunkSize);
758 db.town_name[TOWN_INDEX_TOWN_NAME_SIZE - 1] = '\0'; // make sure string is terminated
759 if (ii22 > 0)
760 {
761 // set "split"-marker
762 db.town_id = 0;
763 db.country_id = 0; // setting this to zero is actually not needed
764 }
765 ib.len = ib.len + sizeof(struct town_index_data_block);
766 fprintf_(stderr, "->block-split: town_id=%lld country_id=%d town_name=%s\n", db.town_id, db.country_id, db.town_name);
767 fwrite(&db, sizeof(struct town_index_data_block), 1, town_index_index_data_block);
768 }
769 fprintf_(stderr, " block-split: END\n");
770 g_free(townname2);
771 }
772 else
773 {
774 strncpy(&db.town_name, townname2, TOWN_INDEX_TOWN_NAME_SIZE);
775 g_free(townname2);
776 db.town_name[TOWN_INDEX_TOWN_NAME_SIZE - 1]= '\0'; // make sure string is terminated
777
778 ib.len = ib.len + sizeof(struct town_index_data_block);
779 fwrite(&db, sizeof(struct town_index_data_block), 1, town_index_index_data_block);
780 }
781
782 if (first == 1)
783 {
784 first = 0;
785 }
786
787 break;
788
789 default:
790 fprintf(stderr, "SQL Error: %d\n", rc);
791 break;
792 }
793 }
794 while (rc == SQLITE_ROW);
795
796 sqlite3_reset(stmt_town_sel005);
797
798 fprintf_(stderr, "end i=%d\n", i);
799 fprintf_(stderr, "end block =%d\n", current_index_block);
800 fprintf_(stderr, "end block old=%d\n", current_index_block_old);
801
802 // rest of the towns
803 if (i > -1)
804 {
805 if (town_index_index_data_block)
806 {
807 fclose(town_index_index_data_block);
808 town_index_index_data_block = NULL;
809 }
810
811 if (USE_STREET_INDEX_COMPRESSION == 1)
812 {
813 #ifdef MAPTOOL_USE_STRINDEX_COMPRESSION
814 ib.len = compress_file(newfilename, newfilename_compr, global_keep_tmpfiles);
815 #endif
816 }
817
818 // append to indexfile
819 fprintf_(stderr, "(last)first_id=%lld offset=%lld len=%lld\n", ib.first_id, ib.offset, ib.len);
820 fwrite(&ib, sizeof(struct town_index_index_block), 1, town_index_index);
821 }
822
823 fprintf_(stderr, "real num of town index blocks=%d\n", current_index_block);
824 fprintf_(stderr, "real size of town index:ftell=%lld\n", (long long)ftello(town_index_index));
825 fprintf_(stderr, "real count index blocks =%lld\n", (long long)is.count_of_index_blocks);
826
827 if (town_index_index != NULL)
828 {
829 fclose(town_index_index);
830 town_index_index = NULL;
831 }
832
833 // write corrections back to file -----------------------------------
834 // write corrections back to file -----------------------------------
835 tempfile_rename("", "town_index_index", "town_index_index_ORIG");
836 FILE *fcorro = tempfile("", "town_index_index_ORIG", 0);
837 FILE *fcorr = tempfile("", "town_index_index", 1);
838 // part 1
839 fwrite(&is, sizeof(struct town_index_index_block_start), 1, fcorr);
840
841 // part 2
842 struct town_index_index_block *ti_ib_mem1 = NULL; // mem pointer, to free the mem
843 struct town_index_index_block *ti_ib1 = NULL; // data pointer, to read data
844 long long s1 = sizeof(struct town_index_index_block) * is.count_of_index_blocks;
845 fprintf_(stderr, "s1 size=%lld\n", (long long)s1);
846 ti_ib_mem1 = g_malloc0(s1);
847 fseeko(fcorro, sizeof(struct town_index_index_block_start), SEEK_SET);
848 fprintf_(stderr, "size of town_index_index_block_start=%lld\n", (long long)sizeof(struct town_index_index_block_start));
849 fread(ti_ib_mem1, sizeof(struct town_index_index_block), is.count_of_index_blocks, fcorro);
850
851 fprintf_(stderr, "read count index blocks =%lld\n", (long long)is.count_of_index_blocks);
852
853 // HINT: ok this is crappy, why don't u fix it if you want? :-)
854 int jj;
855 long long offset_add = 0;
856 for (jj=0;jj<is.count_of_index_blocks;jj++)
857 {
858 ti_ib1 = ti_ib_mem1 + jj;
859
860 // len=266 offset=32
861 fprintf_(stderr, "tblock num =%d len=%lld offset=%lld\n", jj, ti_ib1->len, ti_ib1->offset);
862 // correct offset for all blocks
863 if (jj == 0)
864 {
865 ti_ib1->offset = sizeof(struct town_index_index_block_start) + (is.count_of_index_blocks * sizeof(struct town_index_index_block));
866 offset_add = ti_ib1->offset;
867 }
868 else
869 {
870 ti_ib1->offset = offset_add;
871 }
872 offset_add = offset_add + ti_ib1->len;
873
874 fprintf_(stderr, "tblock num(corr)=%d add=%lld len=%lld offset=%lld\n", jj, offset_add, ti_ib1->len, ti_ib1->offset);
875 }
876
877 fwrite(ti_ib_mem1, sizeof(struct town_index_index_block), is.count_of_index_blocks, fcorr);
878 fclose(fcorr);
879 fclose(fcorro);
880 g_free(ti_ib_mem1);
881 if (global_keep_tmpfiles != 1)
882 {
883 tempfile_unlink("", "town_index_index_ORIG");
884 }
885 // write corrections back to file -----------------------------------
886 // write corrections back to file -----------------------------------
887
888
889
890 if (town_index_index != NULL)
891 {
892 fclose(town_index_index);
893 }
894
895 if (town_index_index_data_block != NULL)
896 {
897 fclose(town_index_index_data_block);
898 }
899
900 if (newfilename)
901 {
902 g_free(newfilename);
903 newfilename = NULL;
904 }
905
906 if (newfilename_compr)
907 {
908 g_free(newfilename_compr);
909 newfilename_compr = NULL;
910 }
911
912
913
914
915
916 // put all parts together
917 town_index_index = tempfile("", "town_index_index", 0);
918 filecopy(town_index_index, out);
919 fclose(town_index_index);
920 if (global_keep_tmpfiles != 1)
921 {
922 tempfile_unlink("", "town_index_index");
923 }
924
925 for (i=1;i < (current_index_block + 2);i++)
926 {
927 fprintf_(stderr, "index block #%d\n", i);
928 newfilename = g_strdup_printf("town_index_index_%d", i);
929 town_index_index_data_block = tempfile("", newfilename, 0);
930
931 if (town_index_index_data_block)
932 {
933 fprintf_(stderr, "using: index block #%d in %s\n", i, newfilename);
934 filecopy(town_index_index_data_block, out);
935 fclose(town_index_index_data_block);
936 if (global_keep_tmpfiles != 1)
937 {
938 tempfile_unlink("", newfilename);
939 }
940 }
941
942 if (newfilename)
943 {
944 g_free(newfilename);
945 newfilename = NULL;
946 }
947 }
948
949 #endif
950 }
951
952
953
954
955
956 void generate_street_index_file(FILE *out)
957 {
958 #ifdef MAPTOOL_USE_SQL
959 int rc = 0;
960 int i;
961 struct streets_index_data_block db;
962 struct streets_index_index_block_start is;
963 struct streets_index_index_block ib;
964 long long last_len = 0;
965 static const char *alpha = "abcdefghijklmnopqrstuvwxyz";
966 char tmp_letter[STREET_INDEX_STREET_NAME_SIZE];
967 char *newfilename = NULL;
968 char *newfilename_compr = NULL;
969 int count_of_blocks;
970 int do_rest;
971 int num1;
972 int num2;
973 char waytype;
974
975 FILE *street_index_index;
976 FILE *street_index_index_data_block;
977
978 // init compression for streets ----------------------
979 // init compression for streets ----------------------
980
981 s_s_outbuf = g_malloc0(s_OUT_BUF_SIZE);
982 /* t_s_outbuf = g_malloc(t_OUT_BUF_SIZE); */
983 s_s_inbuf = g_malloc0(s_IN_BUF_SIZE);
984 /* t_s_inbuf = g_malloc(t_IN_BUF_SIZE); */
985
986 IN_BUF_SIZE = s_IN_BUF_SIZE;
987 s_inbuf = s_s_inbuf;
988 OUT_BUF_SIZE = s_OUT_BUF_SIZE;
989 s_outbuf = s_s_outbuf;
990 COMP_OUT_BUF_SIZE = s_OUT_BUF_SIZE;
991
992 // init compression for streets ----------------------
993 // init compression for streets ----------------------
994
995
996 is.count_of_index_blocks = 703; // 26+1 letters ((26+1)*26 + 1) = 703
997 ib.offset = sizeof (is.count_of_index_blocks) + is.count_of_index_blocks * sizeof(struct streets_index_index_block); // start offset = index size
998 street_index_index = tempfile("", "street_index_index", 1);
999 fprintf_(stderr, "index size=%d\n", ib.offset);
1000
1001 fprintf_(stderr, "ftell=%d\n", ftell(street_index_index));
1002 fwrite(&is, sizeof(struct streets_index_index_block_start), 1, street_index_index);
1003 fprintf_(stderr, "ftell=%d\n", ftell(street_index_index));
1004
1005 // fprintf(stderr, "len=%d\n", strlen(alpha));
1006
1007 count_of_blocks = 702;
1008 do_rest = 0;
1009 num1 = 1;
1010 num2 = 0;
1011 for (i=0; i < count_of_blocks; i++)
1012 {
1013 num2++;
1014 do_rest = 0;
1015 if (num2 == 27)
1016 {
1017 do_rest = 1;
1018 }
1019 else if (num2 > 27)
1020 {
1021 num2 = 1;
1022 num1++;
1023 }
1024
1025 fprintf_(stderr, "i=%d num1=%d num2=%d\n", i, num1, num2);
1026
1027 if (do_rest)
1028 {
1029 sprintf(tmp_letter, "%c", alpha[num1 - 1]);
1030 }
1031 else
1032 {
1033 sprintf(tmp_letter, "%c%c", alpha[num1 - 1], alpha[num2 - 1]);
1034 }
1035 fprintf_(stderr, "letter=%s\n", tmp_letter);
1036
1037 ib.first_letter = alpha[num1 - 1];
1038 ib.len = 0;
1039
1040 newfilename = g_strdup_printf("street_index_index_%d", i);
1041 newfilename_compr = g_strdup_printf("street_index_index_compr_%d", i);
1042 street_index_index_data_block = tempfile("", newfilename, 1);
1043
1044
1045 // ------ TIMER -------
1046 time_t start_tt_4, end_tt_4;
1047 double diff_tt_4;
1048 char outstring_4[200];
1049 // ------ TIMER -------
1050
1051 // ------ TIMER -------
1052 time(&start_tt_4);
1053 // ------ TIMER -------
1054
1055 // loop thru all the streets that match '<letter>...'
1056 sqlite3_bind_text(stmt_sel003, 1, tmp_letter, -1, SQLITE_STATIC);
1057 do
1058 {
1059 rc = sqlite3_step(stmt_sel003);
1060 switch (rc)
1061 {
1062 case SQLITE_DONE:
1063 break;
1064 case SQLITE_ROW:
1065 db.town_id = sqlite3_column_int64(stmt_sel003, 0);
1066 db.lat = transform_from_geo_lat(sqlite3_column_double(stmt_sel003, 1));
1067 db.lon = transform_from_geo_lon(sqlite3_column_double(stmt_sel003, 2));
1068 strncpy(&db.street_name, sqlite3_column_text(stmt_sel003, 3), (STREET_INDEX_STREET_NAME_SIZE - 1));
1069 waytype = sqlite3_column_int(stmt_sel003, 5);
1070 db.street_type = (char)waytype;
1071 db.street_name[(STREET_INDEX_STREET_NAME_SIZE - 2)]= '\0'; // make sure string is terminated
1072
1073 if ((&db.street_name)&&(strlen(&db.street_name) > 1))
1074 {
1075 ib.len = ib.len + sizeof(struct streets_index_data_block);
1076 //fprintf(stderr ,"gen_street_index id:%lld lat:%d lon:%d name:%s\n", db.town_id, db.lat, db.lon, db.street_name);
1077 fwrite(&db, sizeof(struct streets_index_data_block), 1, street_index_index_data_block);
1078 }
1079 else
1080 {
1081 //fprintf(stderr, "streetname1 Error: %s\n", &db.street_name);
1082 }
1083
1084 break;
1085 default:
1086 fprintf(stderr, "SQL Error: %d\n", rc);
1087 break;
1088 }
1089 }
1090 while (rc == SQLITE_ROW);
1091
1092 sqlite3_reset(stmt_sel003);
1093 fclose(street_index_index_data_block);
1094
1095 // ------ TIMER -------
1096 time(&end_tt_4);
1097 diff_tt_4 = difftime(end_tt_4, start_tt_4);
1098 convert_to_human_time(diff_tt_4, outstring_4);
1099 fprintf_(stderr, "-TIME-IND-001: %s\n", outstring_4);
1100 // ------ TIMER -------
1101
1102
1103 // ------ TIMER -------
1104 time(&start_tt_4);
1105 // ------ TIMER -------
1106
1107 sqlite3_bind_int(stmt_sel003u, 1, (i + 1));
1108 sqlite3_bind_text(stmt_sel003u, 2, tmp_letter, -1, SQLITE_STATIC);
1109 sqlite3_step(stmt_sel003u);
1110 sqlite3_reset(stmt_sel003u);
1111 // sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
1112
1113
1114 // ------ TIMER -------
1115 time(&end_tt_4);
1116 diff_tt_4 = difftime(end_tt_4, start_tt_4);
1117 convert_to_human_time(diff_tt_4, outstring_4);
1118 fprintf_(stderr, "-TIME-IND-002: %s\n", outstring_4);
1119 // ------ TIMER -------
1120
1121
1122 // ------ TIMER -------
1123 time(&start_tt_4);
1124 // ------ TIMER -------
1125
1126
1127 if (USE_STREET_INDEX_COMPRESSION == 1)
1128 {
1129 #ifdef MAPTOOL_USE_STRINDEX_COMPRESSION
1130 ib.len = compress_file(newfilename, newfilename_compr, global_keep_tmpfiles);
1131 #endif
1132 }
1133
1134 // ------ TIMER -------
1135 time(&end_tt_4);
1136 diff_tt_4 = difftime(end_tt_4, start_tt_4);
1137 convert_to_human_time(diff_tt_4, outstring_4);
1138 fprintf_(stderr, "-TIME-IND-COMPR: %s\n", outstring_4);
1139 // ------ TIMER -------
1140
1141
1142 last_len = ib.len;
1143 fprintf_(stderr ,"letter=%c offset=%lld len=%lld\n", ib.first_letter, ib.offset, ib.len);
1144 fprintf_(stderr, "ftell=%d\n", ftell(street_index_index));
1145 fwrite(&ib, sizeof(struct streets_index_index_block), 1, street_index_index);
1146 fprintf_(stderr, "ftell=%d\n", ftell(street_index_index));
1147 ib.offset = ib.offset + last_len;
1148
1149 if (newfilename)
1150 {
1151 g_free(newfilename);
1152 newfilename = NULL;
1153 }
1154
1155 if (newfilename_compr)
1156 {
1157 g_free(newfilename_compr);
1158 newfilename_compr = NULL;
1159 }
1160 }
1161
1162 // rest of the streets
1163 fprintf_(stderr, "rest of letters\n");
1164
1165 ib.first_letter = 65; // dummy "A"
1166 ib.len = 0;
1167
1168 newfilename = g_strdup_printf("street_index_index_%d", i);
1169 newfilename_compr = g_strdup_printf("street_index_index_compr_%d", i);
1170 street_index_index_data_block = tempfile("", newfilename, 1);
1171
1172 do
1173 {
1174 rc = sqlite3_step(stmt_sel004);
1175 switch (rc)
1176 {
1177 case SQLITE_DONE:
1178 break;
1179 case SQLITE_ROW:
1180 db.town_id = sqlite3_column_int64(stmt_sel004, 0);
1181 db.lat = transform_from_geo_lat(sqlite3_column_double(stmt_sel004, 1));
1182 db.lon = transform_from_geo_lon(sqlite3_column_double(stmt_sel004, 2));
1183
1184 strncpy(&db.street_name, sqlite3_column_text(stmt_sel004, 3), (STREET_INDEX_STREET_NAME_SIZE - 1));
1185 waytype = sqlite3_column_int(stmt_sel004, 5);
1186 db.street_type = (char)waytype;
1187 db.street_name[(STREET_INDEX_STREET_NAME_SIZE - 2)]= '\0'; // make sure string is terminated
1188
1189 if ((&db.street_name)&&(strlen(&db.street_name) > 1))
1190 {
1191 ib.len = ib.len + sizeof(struct streets_index_data_block);
1192 // fprintf(stderr ,"id:%lld lat:%d lon:%d name:%s waytype:%d\n", db.town_id, db.lat, db.lon, db.street_name, waytype);
1193 fwrite(&db, sizeof(struct streets_index_data_block), 1, street_index_index_data_block);
1194 }
1195 else
1196 {
1197 //fprintf(stderr, "streetname2 Error: %s\n", &db.street_name);
1198 }
1199
1200 break;
1201
1202 default:
1203 fprintf(stderr, "SQL Error: %d\n", rc);
1204 break;
1205 }
1206 }
1207 while (rc == SQLITE_ROW);
1208 sqlite3_reset(stmt_sel004);
1209 fclose(street_index_index_data_block);
1210
1211 if (USE_STREET_INDEX_COMPRESSION == 1)
1212 {
1213 #ifdef MAPTOOL_USE_STRINDEX_COMPRESSION
1214 ib.len = compress_file(newfilename, newfilename_compr, global_keep_tmpfiles);
1215 #endif
1216 }
1217
1218 last_len = ib.len;
1219 fprintf_(stderr ,"letter=%c offset=%lld len=%lld\n", ib.first_letter, ib.offset, ib.len);
1220 fprintf_(stderr, "ftell=%d\n", ftell(street_index_index));
1221 fwrite(&ib, sizeof(struct streets_index_index_block), 1, street_index_index);
1222 fprintf_(stderr, "ftell=%d\n", ftell(street_index_index));
1223 // ib.offset = ib.offset + last_len;
1224
1225
1226 fclose(street_index_index);
1227
1228 if (newfilename)
1229 {
1230 g_free(newfilename);
1231 newfilename = NULL;
1232 }
1233
1234 if (newfilename_compr)
1235 {
1236 g_free(newfilename_compr);
1237 newfilename_compr = NULL;
1238 }
1239
1240 // put all parts together
1241 street_index_index = tempfile("", "street_index_index", 0);
1242 filecopy(street_index_index, out);
1243 fclose(street_index_index);
1244
1245 if (global_keep_tmpfiles != 1)
1246 {
1247 tempfile_unlink("", "street_index_index");
1248 }
1249
1250 for (i=0;i < (is.count_of_index_blocks + 1);i++)
1251 {
1252 //fprintf(stderr, "cat #%d\n", i);
1253 newfilename = g_strdup_printf("street_index_index_%d", i);
1254 street_index_index_data_block = tempfile("", newfilename, 0);
1255
1256 if (street_index_index_data_block)
1257 {
1258 filecopy(street_index_index_data_block, out);
1259 fclose(street_index_index_data_block);
1260 if (global_keep_tmpfiles != 1)
1261 {
1262 tempfile_unlink("", newfilename);
1263 }
1264 }
1265
1266 if (newfilename)
1267 {
1268 g_free(newfilename);
1269 newfilename = NULL;
1270 }
1271 }
1272 #endif
1273 }
1274

   
Visit the ZANavi Wiki