/[zanavi_public1]/navit/navit/maptool/maptool.c
ZANavi

Contents of /navit/navit/maptool/maptool.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 8 - (show annotations) (download)
Fri Oct 28 21:39:42 2011 UTC (12 years, 5 months ago) by zoff99
File MIME type: text/plain
File size: 17656 byte(s)
import
1 /**
2 * Navit, a modular navigation system.
3 * Copyright (C) 2005-2011 Navit Team
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * version 2 as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20
21 #define _FILE_OFFSET_BITS 64
22 #define _LARGEFILE_SOURCE
23 #define _LARGEFILE64_SOURCE
24 #include <stdlib.h>
25 #include <glib.h>
26 #include <assert.h>
27 #include <string.h>
28 #include <signal.h>
29 #include <stdio.h>
30 #include <math.h>
31 #include <getopt.h>
32 #include <unistd.h>
33 #include <fcntl.h>
34 #include <sys/stat.h>
35 #include <zlib.h>
36 #include "file.h"
37 #include "item.h"
38 #include "map.h"
39 #include "zipfile.h"
40 #include "main.h"
41 #include "config.h"
42 #include "linguistics.h"
43 #include "plugin.h"
44 #include "util.h"
45 #include "maptool.h"
46
47 long long slice_size=1024*1024*1024;
48 int attr_debug_level=1;
49 int ignore_unkown = 0;
50 GHashTable *dedupe_ways_hash;
51 int phase;
52 int slices;
53 struct buffer node_buffer = {
54 64*1024*1024,
55 };
56
57 int processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles;
58
59 int overlap=1;
60
61 int bytes_read;
62
63
64
65 void
66 sig_alrm(int sig)
67 {
68 #ifndef _WIN32
69 signal(SIGALRM, sig_alrm);
70 alarm(30);
71 #endif
72 fprintf(stderr,"PROGRESS%d: Processed %d nodes (%d out) %d ways %d relations %d tiles\n", phase, processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles);
73 }
74
75 void
76 sig_alrm_end(void)
77 {
78 #ifndef _WIN32
79 alarm(0);
80 #endif
81 }
82
83 static struct plugins *plugins;
84
85 static void add_plugin(char *path)
86 {
87 struct attr **attrs;
88
89 if (! plugins)
90 plugins=plugins_new();
91 attrs=(struct attr*[]){&(struct attr){attr_path,{path}},NULL};
92 plugin_new(&(struct attr){attr_plugins,.u.plugins=plugins}, attrs);
93 }
94
95 static void
96 maptool_init(FILE* rule_file)
97 {
98 if (plugins)
99 plugins_init(plugins);
100 osm_init(rule_file);
101 }
102
103 static void
104 usage(FILE *f)
105 {
106 /* DEVELOPPERS : don't forget to update the manpage if you modify theses options */
107 fprintf(f,"\n");
108 fprintf(f,"maptool - parse osm textfile and converts to Navit binfile format\n\n");
109 fprintf(f,"Usage :\n");
110 fprintf(f,"bzcat planet.osm.bz2 | maptool mymap.bin\n");
111 fprintf(f,"Available switches:\n");
112 fprintf(f,"-h (--help) : this screen\n");
113 fprintf(f,"-5 (--md5) : set file where to write md5 sum\n");
114 fprintf(f,"-6 (--64bit) : set zip 64 bit compression\n");
115 fprintf(f,"-a (--attr-debug-level) : control which data is included in the debug attribute\n");
116 fprintf(f,"-c (--dump-coordinates) : dump coordinates after phase 1\n");
117 #ifdef HAVE_POSTGRESQL
118 fprintf(f,"-d (--db) : get osm data out of a postgresql database with osm simple scheme and given connect string\n");
119 #endif
120 fprintf(f,"-e (--end) : end at specified phase\n");
121 fprintf(f,"-i (--input-file) : specify the input file name (OSM), overrules default stdin\n");
122 fprintf(f,"-k (--keep-tmpfiles) : do not delete tmp files after processing. useful to reuse them\n\n");
123 fprintf(f,"-N (--nodes-only) : process only nodes\n");
124 fprintf(f,"-o (--coverage) : map every street to item coverage\n");
125 fprintf(f,"-P (--protobuf) : input file is protobuf\n");
126 fprintf(f,"-r (--rule-file) : read mapping rules from specified file\n");
127 fprintf(f,"-s (--start) : start at specified phase\n");
128 fprintf(f,"-S (--slice-size) : defines the amount of memory to use, in bytes. Default is 1GB\n");
129 fprintf(f,"-w (--dedupe-ways) : ensure no duplicate ways or nodes. useful when using several input files\n");
130 fprintf(f,"-W (--ways-only) : process only ways\n");
131 fprintf(f,"-z (--compression-level) : set the compression level\n");
132
133 exit(1);
134 }
135
136 int main(int argc, char **argv)
137 {
138 FILE *ways=NULL,*ways_split=NULL,*ways_split_index=NULL,*nodes=NULL,*turn_restrictions=NULL,*graph=NULL,*coastline=NULL,*tilesdir,*coords,*relations=NULL,*boundaries=NULL;
139 FILE *files[10];
140 FILE *references[10];
141
142 #if 0
143 char *map=g_strdup(attrmap);
144 #endif
145 int zipnum,c,start=1,end=99,dump_coordinates=0;
146 int keep_tmpfiles=0;
147 int process_nodes=1, process_ways=1, process_relations=1;
148 #ifdef HAVE_ZLIB
149 int compression_level=9;
150 #else
151 int compression_level=0;
152 #endif
153 int zip64=0;
154 int output=0;
155 int input=0;
156 int protobuf=0;
157 int f,pos;
158 char *result,*optarg_cp,*attr_name,*attr_value;
159 char *protobufdb=NULL,*protobufdb_operation=NULL,*md5file=NULL;
160 #ifdef HAVE_POSTGRESQL
161 char *dbstr=NULL;
162 #endif
163
164 FILE* input_file = stdin; // input data
165
166 FILE* rule_file = NULL; // external rule file
167
168 struct attr *attrs[10];
169 GList *map_handles=NULL;
170 struct map *handle;
171 #if 0
172 char *suffixes[]={"m0l0", "m0l1","m0l2","m0l3","m0l4","m0l5","m0l6"};
173 char *suffixes[]={"m","r"};
174 #else
175 char *suffixes[]={""};
176 #endif
177 char *suffix=suffixes[0];
178
179 int suffix_count=sizeof(suffixes)/sizeof(char *);
180 int i;
181 char r[] ="r"; /* Used to make compiler happy due to Bug 35903 in gcc */
182 main_init(argv[0]);
183 struct zip_info *zip_info=NULL;
184 int suffix_start=0;
185 char *timestamp=current_to_iso8601();
186 char *url=NULL;
187 #ifndef HAVE_GLIB
188 _g_slice_thread_init_nomessage();
189 #endif
190
191 while (1) {
192 #if 0
193 int this_option_optind = optind ? optind : 1;
194 #endif
195 int option_index = 0;
196 static struct option long_options[] = {
197 {"md5", 1, 0, '5'},
198 {"64bit", 0, 0, '6'},
199 {"attr-debug-level", 1, 0, 'a'},
200 {"binfile", 0, 0, 'b'},
201 {"compression-level", 1, 0, 'z'},
202 #ifdef HAVE_POSTGRESQL
203 {"db", 1, 0, 'd'},
204 #endif
205 {"dedupe-ways", 0, 0, 'w'},
206 {"dump", 0, 0, 'D'},
207 {"dump-coordinates", 0, 0, 'c'},
208 {"end", 1, 0, 'e'},
209 {"help", 0, 0, 'h'},
210 {"keep-tmpfiles", 0, 0, 'k'},
211 {"nodes-only", 0, 0, 'N'},
212 {"map", 1, 0, 'm'},
213 {"plugin", 1, 0, 'p'},
214 {"protobuf", 0, 0, 'P'},
215 {"start", 1, 0, 's'},
216 {"input-file", 1, 0, 'i'},
217 {"rule-file", 1, 0, 'r'},
218 {"ignore-unknown", 0, 0, 'n'},
219 {"url", 1, 0, 'u'},
220 {"ways-only", 0, 0, 'W'},
221 {"slice-size", 1, 0, 'S'},
222 {0, 0, 0, 0}
223 };
224 c = getopt_long (argc, argv, "5:6B:DNO:PWS:a:bc"
225 #ifdef HAVE_POSTGRESQL
226 "d:"
227 #endif
228 "e:hi:knm:p:r:s:wu:z:", long_options, &option_index);
229 if (c == -1)
230 break;
231 switch (c) {
232 case '5':
233 md5file=optarg;
234 break;
235 case '6':
236 zip64=1;
237 break;
238 case 'B':
239 protobufdb=optarg;
240 break;
241 case 'D':
242 output=1;
243 break;
244 case 'N':
245 process_ways=0;
246 break;
247 case 'R':
248 process_relations=0;
249 break;
250 case 'O':
251 protobufdb_operation=optarg;
252 output=1;
253 break;
254 case 'P':
255 protobuf=1;
256 break;
257 case 'S':
258 slice_size=atoll(optarg);
259 break;
260 case 'W':
261 process_nodes=0;
262 break;
263 case 'a':
264 attr_debug_level=atoi(optarg);
265 break;
266 case 'b':
267 input=1;
268 break;
269 case 'c':
270 dump_coordinates=1;
271 break;
272 #ifdef HAVE_POSTGRESQL
273 case 'd':
274 dbstr=optarg;
275 break;
276 #endif
277 case 'e':
278 end=atoi(optarg);
279 break;
280 case 'h':
281 usage(stdout);
282 break;
283 case 'm':
284 optarg_cp=g_strdup(optarg);
285 pos=0;
286 i=0;
287 attr_name=g_strdup(optarg);
288 attr_value=g_strdup(optarg);
289 while (i < 9 && attr_from_line(optarg_cp, NULL, &pos, attr_value, attr_name)) {
290 attrs[i]=attr_new_from_text(attr_name,attr_value);
291 if (attrs[i]) {
292 i++;
293 } else {
294 fprintf(stderr,"Failed to convert %s=%s to attribute\n",attr_name,attr_value);
295 }
296 attr_value=g_strdup(optarg);
297 }
298 attrs[i++]=NULL;
299 g_free(attr_value);
300 g_free(optarg_cp);
301 handle=map_new(NULL, attrs);
302 if (! handle) {
303 fprintf(stderr,"Failed to create map from attributes\n");
304 exit(1);
305 }
306 map_handles=g_list_append(map_handles,handle);
307 break;
308 case 'n':
309 fprintf(stderr,"I will IGNORE unknown types\n");
310 ignore_unkown=1;
311 break;
312 case 'k':
313 fprintf(stderr,"I will KEEP tmp files\n");
314 keep_tmpfiles=1;
315 break;
316 case 'p':
317 add_plugin(optarg);
318 break;
319 case 's':
320 start=atoi(optarg);
321 break;
322 case 'w':
323 dedupe_ways_hash=g_hash_table_new(NULL, NULL);
324 break;
325 case 'i':
326 input_file = fopen( optarg, "r" );
327 if ( input_file == NULL )
328 {
329 fprintf( stderr, "\nInput file (%s) not found\n", optarg );
330 exit( -1 );
331 }
332 break;
333 case 'r':
334 rule_file = fopen( optarg, "r" );
335 if ( rule_file == NULL )
336 {
337 fprintf( stderr, "\nRule file (%s) not found\n", optarg );
338 exit( -1 );
339 }
340 break;
341 case 'u':
342 url=optarg;
343 break;
344 #ifdef HAVE_ZLIB
345 case 'z':
346 compression_level=atoi(optarg);
347 break;
348 #endif
349 case '?':
350 usage(stderr);
351 break;
352 default:
353 fprintf(stderr,"c=%d\n", c);
354 }
355
356 }
357 if (optind != argc-(output == 1 ? 0:1))
358 usage(stderr);
359 result=argv[optind];
360
361 // initialize plugins and OSM mappings
362 maptool_init(rule_file);
363 if (protobufdb_operation) {
364 osm_protobufdb_load(input_file, protobufdb);
365 return 0;
366 }
367
368 // input from an OSM file
369 if (input == 0) {
370 if (start == 1) {
371 unlink("coords.tmp");
372 if (process_ways)
373 ways=tempfile(suffix,"ways",1);
374 if (process_nodes)
375 nodes=tempfile(suffix,"nodes",1);
376 if (process_ways && process_nodes)
377 turn_restrictions=tempfile(suffix,"turn_restrictions",1);
378 if (process_relations)
379 boundaries=tempfile(suffix,"boundaries",1);
380 phase=1;
381 fprintf(stderr,"PROGRESS: Phase 1: collecting data\n");
382 #ifdef HAVE_POSTGRESQL
383 if (dbstr)
384 map_collect_data_osm_db(dbstr,ways,nodes,turn_restrictions,boundaries);
385 else
386 #endif
387 if (map_handles) {
388 GList *l;
389 phase1_map(map_handles,ways,nodes);
390 l=map_handles;
391 while (l) {
392 map_destroy(l->data);
393 l=g_list_next(l);
394 }
395 }
396 else if (protobuf)
397 map_collect_data_osm_protobuf(input_file,ways,nodes,turn_restrictions,boundaries);
398 else
399 map_collect_data_osm(input_file,ways,nodes,turn_restrictions,boundaries);
400 if (slices) {
401 fprintf(stderr,"%d slices\n",slices);
402 flush_nodes(1);
403 for (i = slices-2 ; i>=0 ; i--) {
404 fprintf(stderr, "slice %d of %d\n",slices-i-1,slices-1);
405 load_buffer("coords.tmp",&node_buffer, i*slice_size, slice_size);
406 resolve_ways(ways, NULL);
407 save_buffer("coords.tmp",&node_buffer, i*slice_size);
408 }
409 } else
410 save_buffer("coords.tmp",&node_buffer, 0);
411 if (ways)
412 fclose(ways);
413 if (nodes)
414 fclose(nodes);
415 if (turn_restrictions)
416 fclose(turn_restrictions);
417 if (boundaries)
418 fclose(boundaries);
419 }
420 if (!slices) {
421 if (end == 1 || dump_coordinates)
422 flush_nodes(1);
423 else
424 slices++;
425 }
426 if (end == 1)
427 exit(0);
428 if (start == 2) {
429 load_buffer("coords.tmp",&node_buffer,0, slice_size);
430 }
431 if (start <= 2) {
432 if (process_ways) {
433 ways=tempfile(suffix,"ways",0);
434 phase=2;
435 fprintf(stderr,"PROGRESS: Phase 2: finding intersections\n");
436 for (i = 0 ; i < slices ; i++) {
437 int final=(i >= slices-1);
438 ways_split=tempfile(suffix,"ways_split",1);
439 ways_split_index=final ? tempfile(suffix,"ways_split_index",1) : NULL;
440 graph=tempfile(suffix,"graph",1);
441 coastline=tempfile(suffix,"coastline",1);
442 if (i)
443 load_buffer("coords.tmp",&node_buffer, i*slice_size, slice_size);
444 map_find_intersections(ways,ways_split,ways_split_index,graph,coastline,final);
445 fclose(ways_split);
446 if (ways_split_index)
447 fclose(ways_split_index);
448 fclose(ways);
449 fclose(graph);
450 fclose(coastline);
451 if (! final) {
452 tempfile_rename(suffix,"ways_split","ways_to_resolve");
453 ways=tempfile(suffix,"ways_to_resolve",0);
454 }
455 }
456 if(!keep_tmpfiles)
457 tempfile_unlink(suffix,"ways");
458 tempfile_unlink(suffix,"ways_to_resolve");
459 } else
460 fprintf(stderr,"PROGRESS: Skipping Phase 2\n");
461 }
462 free(node_buffer.base);
463 node_buffer.base=NULL;
464 node_buffer.malloced=0;
465 node_buffer.size=0;
466 if (end == 2)
467 exit(0);
468 } else {
469 ways_split=tempfile(suffix,"ways_split",0);
470 process_binfile(stdin, ways_split);
471 fclose(ways_split);
472 }
473
474 #if 1
475 coastline=tempfile(suffix,"coastline",0);
476 if (coastline) {
477 ways_split=tempfile(suffix,"ways_split",2);
478 fprintf(stderr,"coastline=%p\n",coastline);
479 process_coastlines(coastline, ways_split);
480 fclose(ways_split);
481 fclose(coastline);
482 }
483 #endif
484 if (start <= 3) {
485 fprintf(stderr,"PROGRESS: Phase 3: sorting countries, generating turn restrictions\n");
486 sort_countries(keep_tmpfiles);
487 if (process_relations) {
488 #if 0
489 boundaries=tempfile(suffix,"boundaries",0);
490 ways_split=tempfile(suffix,"ways_split",0);
491 process_boundaries(boundaries, ways_split);
492 fclose(boundaries);
493 fclose(ways_split);
494 exit(0);
495 #endif
496 turn_restrictions=tempfile(suffix,"turn_restrictions",0);
497 if (turn_restrictions) {
498 relations=tempfile(suffix,"relations",1);
499 coords=fopen("coords.tmp","rb");
500 ways_split=tempfile(suffix,"ways_split",0);
501 ways_split_index=tempfile(suffix,"ways_split_index",0);
502 process_turn_restrictions(turn_restrictions,coords,ways_split,ways_split_index,relations);
503 fclose(ways_split_index);
504 fclose(ways_split);
505 fclose(coords);
506 fclose(relations);
507 fclose(turn_restrictions);
508 if(!keep_tmpfiles)
509 tempfile_unlink(suffix,"turn_restrictions");
510 }
511 }
512 if(!keep_tmpfiles)
513 tempfile_unlink(suffix,"ways_split_index");
514 }
515 if (end == 3)
516 exit(0);
517 if (output == 1) {
518 fprintf(stderr,"PROGRESS: Phase 4: dumping\n");
519 if (process_nodes) {
520 nodes=tempfile(suffix,"nodes",0);
521 if (nodes) {
522 dump(nodes);
523 fclose(nodes);
524 }
525 }
526 if (process_ways) {
527 ways_split=tempfile(suffix,"ways_split",0);
528 if (ways_split) {
529 dump(ways_split);
530 fclose(ways_split);
531 }
532 }
533 if (process_relations) {
534 relations=tempfile(suffix,"relations",0);
535 fprintf(stderr,"Relations=%p\n",relations);
536 if (relations) {
537 dump(relations);
538 fclose(relations);
539 }
540 }
541 exit(0);
542 }
543 for (i = suffix_start ; i < suffix_count ; i++) {
544 suffix=suffixes[i];
545 if (start <= 4) {
546 phase=3;
547 if (i == suffix_start) {
548 zip_info=zip_new();
549 zip_set_zip64(zip_info, zip64);
550 zip_set_timestamp(zip_info, timestamp);
551 }
552 zipnum=zip_get_zipnum(zip_info);
553 fprintf(stderr,"PROGRESS: Phase 4: generating tiles %s\n",suffix);
554 tilesdir=tempfile(suffix,"tilesdir",1);
555 if (!strcmp(suffix,r)) { /* Makes compiler happy due to bug 35903 in gcc */
556 ch_generate_tiles(suffixes[0],suffix,tilesdir,zip_info);
557 } else {
558 for (f = 0 ; f < 3 ; f++)
559 files[f]=NULL;
560 if (process_relations)
561 files[0]=tempfile(suffix,"relations",0);
562 if (process_ways)
563 files[1]=tempfile(suffix,"ways_split",0);
564 if (process_nodes)
565 files[2]=tempfile(suffix,"nodes",0);
566 phase4(files,3,0,suffix,tilesdir,zip_info);
567 for (f = 0 ; f < 3 ; f++) {
568 if (files[f])
569 fclose(files[f]);
570 }
571 }
572 fclose(tilesdir);
573 zip_set_zipnum(zip_info,zipnum);
574 }
575 if (end == 4)
576 exit(0);
577 if (zip_info) {
578 zip_destroy(zip_info);
579 zip_info=NULL;
580 }
581 if (start <= 5) {
582 phase=4;
583 fprintf(stderr,"PROGRESS: Phase 5: assembling map %s\n",suffix);
584 if (i == suffix_start) {
585 char *zipdir=tempfile_name("zipdir","");
586 char *zipindex=tempfile_name("index","");
587 zip_info=zip_new();
588 zip_set_zip64(zip_info, zip64);
589 zip_set_timestamp(zip_info, timestamp);
590 zip_set_maxnamelen(zip_info, 14+strlen(suffixes[0]));
591 zip_set_compression_level(zip_info, compression_level);
592 if (md5file)
593 zip_set_md5(zip_info, 1);
594 zip_open(zip_info, result, zipdir, zipindex);
595 if (url) {
596 map_information_attrs[1].type=attr_url;
597 map_information_attrs[1].u.str=url;
598 }
599 index_init(zip_info, 1);
600 }
601 if (!strcmp(suffix,r)) { /* Makes compiler happy due to bug 35903 in gcc */
602 ch_assemble_map(suffixes[0],suffix,zip_info);
603 } else {
604 for (f = 0 ; f < 3 ; f++) {
605 files[f]=NULL;
606 references[f]=NULL;
607 }
608 if (process_relations)
609 files[0]=tempfile(suffix,"relations",0);
610 if (process_ways) {
611 files[1]=tempfile(suffix,"ways_split",0);
612 references[1]=tempfile(suffix,"ways_split_ref",1);
613 }
614 if (process_nodes)
615 files[2]=tempfile(suffix,"nodes",0);
616 fprintf(stderr,"Slice %d\n",i);
617
618 phase5(files,references,3,0,suffix,zip_info);
619 for (f = 0 ; f < 3 ; f++) {
620 if (files[f])
621 fclose(files[f]);
622 if (references[f])
623 fclose(references[f]);
624 }
625 }
626 if(!keep_tmpfiles) {
627 tempfile_unlink(suffix,"relations");
628 tempfile_unlink(suffix,"nodes");
629 tempfile_unlink(suffix,"ways_split");
630 tempfile_unlink(suffix,"ways_split_ref");
631 tempfile_unlink(suffix,"coastline");
632 tempfile_unlink(suffix,"turn_restrictions");
633 tempfile_unlink(suffix,"graph");
634 tempfile_unlink(suffix,"tilesdir");
635 tempfile_unlink(suffix,"boundaries");
636 unlink("coords.tmp");
637 }
638 if (i == suffix_count-1) {
639 unsigned char md5_data[16];
640 zipnum=zip_get_zipnum(zip_info);
641 add_aux_tiles("auxtiles.txt", zip_info);
642 write_countrydir(zip_info);
643 zip_set_zipnum(zip_info, zipnum);
644 write_aux_tiles(zip_info);
645 zip_write_index(zip_info);
646 zip_write_directory(zip_info);
647 zip_close(zip_info);
648 if (md5file && zip_get_md5(zip_info, md5_data)) {
649 FILE *md5=fopen(md5file,"w");
650 int i;
651 for (i = 0 ; i < 16 ; i++)
652 fprintf(md5,"%02x",md5_data[i]);
653 fprintf(md5,"\n");
654 fclose(md5);
655 }
656 if (!keep_tmpfiles) {
657 remove_countryfiles();
658 tempfile_unlink("index","");
659 tempfile_unlink("zipdir","");
660 }
661 }
662 }
663 }
664 return 0;
665 }

   
Visit the ZANavi Wiki