/[zanavi_public1]/navit/navit/maptool/maptool.c
ZANavi

Contents of /navit/navit/maptool/maptool.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 8 - (hide annotations) (download)
Fri Oct 28 21:39:42 2011 UTC (9 years, 5 months ago) by zoff99
File MIME type: text/plain
File size: 17656 byte(s)
import
1 zoff99 8 /**
2     * Navit, a modular navigation system.
3     * Copyright (C) 2005-2011 Navit Team
4     *
5     * This program is free software; you can redistribute it and/or
6     * modify it under the terms of the GNU General Public License
7     * version 2 as published by the Free Software Foundation.
8     *
9     * This program is distributed in the hope that it will be useful,
10     * but WITHOUT ANY WARRANTY; without even the implied warranty of
11     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12     * GNU General Public License for more details.
13     *
14     * You should have received a copy of the GNU General Public License
15     * along with this program; if not, write to the
16     * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17     * Boston, MA 02110-1301, USA.
18     */
19    
20    
21     #define _FILE_OFFSET_BITS 64
22     #define _LARGEFILE_SOURCE
23     #define _LARGEFILE64_SOURCE
24     #include <stdlib.h>
25     #include <glib.h>
26     #include <assert.h>
27     #include <string.h>
28     #include <signal.h>
29     #include <stdio.h>
30     #include <math.h>
31     #include <getopt.h>
32     #include <unistd.h>
33     #include <fcntl.h>
34     #include <sys/stat.h>
35     #include <zlib.h>
36     #include "file.h"
37     #include "item.h"
38     #include "map.h"
39     #include "zipfile.h"
40     #include "main.h"
41     #include "config.h"
42     #include "linguistics.h"
43     #include "plugin.h"
44     #include "util.h"
45     #include "maptool.h"
46    
47     long long slice_size=1024*1024*1024;
48     int attr_debug_level=1;
49     int ignore_unkown = 0;
50     GHashTable *dedupe_ways_hash;
51     int phase;
52     int slices;
53     struct buffer node_buffer = {
54     64*1024*1024,
55     };
56    
57     int processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles;
58    
59     int overlap=1;
60    
61     int bytes_read;
62    
63    
64    
65     void
66     sig_alrm(int sig)
67     {
68     #ifndef _WIN32
69     signal(SIGALRM, sig_alrm);
70     alarm(30);
71     #endif
72     fprintf(stderr,"PROGRESS%d: Processed %d nodes (%d out) %d ways %d relations %d tiles\n", phase, processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles);
73     }
74    
75     void
76     sig_alrm_end(void)
77     {
78     #ifndef _WIN32
79     alarm(0);
80     #endif
81     }
82    
83     static struct plugins *plugins;
84    
85     static void add_plugin(char *path)
86     {
87     struct attr **attrs;
88    
89     if (! plugins)
90     plugins=plugins_new();
91     attrs=(struct attr*[]){&(struct attr){attr_path,{path}},NULL};
92     plugin_new(&(struct attr){attr_plugins,.u.plugins=plugins}, attrs);
93     }
94    
95     static void
96     maptool_init(FILE* rule_file)
97     {
98     if (plugins)
99     plugins_init(plugins);
100     osm_init(rule_file);
101     }
102    
103     static void
104     usage(FILE *f)
105     {
106     /* DEVELOPPERS : don't forget to update the manpage if you modify theses options */
107     fprintf(f,"\n");
108     fprintf(f,"maptool - parse osm textfile and converts to Navit binfile format\n\n");
109     fprintf(f,"Usage :\n");
110     fprintf(f,"bzcat planet.osm.bz2 | maptool mymap.bin\n");
111     fprintf(f,"Available switches:\n");
112     fprintf(f,"-h (--help) : this screen\n");
113     fprintf(f,"-5 (--md5) : set file where to write md5 sum\n");
114     fprintf(f,"-6 (--64bit) : set zip 64 bit compression\n");
115     fprintf(f,"-a (--attr-debug-level) : control which data is included in the debug attribute\n");
116     fprintf(f,"-c (--dump-coordinates) : dump coordinates after phase 1\n");
117     #ifdef HAVE_POSTGRESQL
118     fprintf(f,"-d (--db) : get osm data out of a postgresql database with osm simple scheme and given connect string\n");
119     #endif
120     fprintf(f,"-e (--end) : end at specified phase\n");
121     fprintf(f,"-i (--input-file) : specify the input file name (OSM), overrules default stdin\n");
122     fprintf(f,"-k (--keep-tmpfiles) : do not delete tmp files after processing. useful to reuse them\n\n");
123     fprintf(f,"-N (--nodes-only) : process only nodes\n");
124     fprintf(f,"-o (--coverage) : map every street to item coverage\n");
125     fprintf(f,"-P (--protobuf) : input file is protobuf\n");
126     fprintf(f,"-r (--rule-file) : read mapping rules from specified file\n");
127     fprintf(f,"-s (--start) : start at specified phase\n");
128     fprintf(f,"-S (--slice-size) : defines the amount of memory to use, in bytes. Default is 1GB\n");
129     fprintf(f,"-w (--dedupe-ways) : ensure no duplicate ways or nodes. useful when using several input files\n");
130     fprintf(f,"-W (--ways-only) : process only ways\n");
131     fprintf(f,"-z (--compression-level) : set the compression level\n");
132    
133     exit(1);
134     }
135    
136     int main(int argc, char **argv)
137     {
138     FILE *ways=NULL,*ways_split=NULL,*ways_split_index=NULL,*nodes=NULL,*turn_restrictions=NULL,*graph=NULL,*coastline=NULL,*tilesdir,*coords,*relations=NULL,*boundaries=NULL;
139     FILE *files[10];
140     FILE *references[10];
141    
142     #if 0
143     char *map=g_strdup(attrmap);
144     #endif
145     int zipnum,c,start=1,end=99,dump_coordinates=0;
146     int keep_tmpfiles=0;
147     int process_nodes=1, process_ways=1, process_relations=1;
148     #ifdef HAVE_ZLIB
149     int compression_level=9;
150     #else
151     int compression_level=0;
152     #endif
153     int zip64=0;
154     int output=0;
155     int input=0;
156     int protobuf=0;
157     int f,pos;
158     char *result,*optarg_cp,*attr_name,*attr_value;
159     char *protobufdb=NULL,*protobufdb_operation=NULL,*md5file=NULL;
160     #ifdef HAVE_POSTGRESQL
161     char *dbstr=NULL;
162     #endif
163    
164     FILE* input_file = stdin; // input data
165    
166     FILE* rule_file = NULL; // external rule file
167    
168     struct attr *attrs[10];
169     GList *map_handles=NULL;
170     struct map *handle;
171     #if 0
172     char *suffixes[]={"m0l0", "m0l1","m0l2","m0l3","m0l4","m0l5","m0l6"};
173     char *suffixes[]={"m","r"};
174     #else
175     char *suffixes[]={""};
176     #endif
177     char *suffix=suffixes[0];
178    
179     int suffix_count=sizeof(suffixes)/sizeof(char *);
180     int i;
181     char r[] ="r"; /* Used to make compiler happy due to Bug 35903 in gcc */
182     main_init(argv[0]);
183     struct zip_info *zip_info=NULL;
184     int suffix_start=0;
185     char *timestamp=current_to_iso8601();
186     char *url=NULL;
187     #ifndef HAVE_GLIB
188     _g_slice_thread_init_nomessage();
189     #endif
190    
191     while (1) {
192     #if 0
193     int this_option_optind = optind ? optind : 1;
194     #endif
195     int option_index = 0;
196     static struct option long_options[] = {
197     {"md5", 1, 0, '5'},
198     {"64bit", 0, 0, '6'},
199     {"attr-debug-level", 1, 0, 'a'},
200     {"binfile", 0, 0, 'b'},
201     {"compression-level", 1, 0, 'z'},
202     #ifdef HAVE_POSTGRESQL
203     {"db", 1, 0, 'd'},
204     #endif
205     {"dedupe-ways", 0, 0, 'w'},
206     {"dump", 0, 0, 'D'},
207     {"dump-coordinates", 0, 0, 'c'},
208     {"end", 1, 0, 'e'},
209     {"help", 0, 0, 'h'},
210     {"keep-tmpfiles", 0, 0, 'k'},
211     {"nodes-only", 0, 0, 'N'},
212     {"map", 1, 0, 'm'},
213     {"plugin", 1, 0, 'p'},
214     {"protobuf", 0, 0, 'P'},
215     {"start", 1, 0, 's'},
216     {"input-file", 1, 0, 'i'},
217     {"rule-file", 1, 0, 'r'},
218     {"ignore-unknown", 0, 0, 'n'},
219     {"url", 1, 0, 'u'},
220     {"ways-only", 0, 0, 'W'},
221     {"slice-size", 1, 0, 'S'},
222     {0, 0, 0, 0}
223     };
224     c = getopt_long (argc, argv, "5:6B:DNO:PWS:a:bc"
225     #ifdef HAVE_POSTGRESQL
226     "d:"
227     #endif
228     "e:hi:knm:p:r:s:wu:z:", long_options, &option_index);
229     if (c == -1)
230     break;
231     switch (c) {
232     case '5':
233     md5file=optarg;
234     break;
235     case '6':
236     zip64=1;
237     break;
238     case 'B':
239     protobufdb=optarg;
240     break;
241     case 'D':
242     output=1;
243     break;
244     case 'N':
245     process_ways=0;
246     break;
247     case 'R':
248     process_relations=0;
249     break;
250     case 'O':
251     protobufdb_operation=optarg;
252     output=1;
253     break;
254     case 'P':
255     protobuf=1;
256     break;
257     case 'S':
258     slice_size=atoll(optarg);
259     break;
260     case 'W':
261     process_nodes=0;
262     break;
263     case 'a':
264     attr_debug_level=atoi(optarg);
265     break;
266     case 'b':
267     input=1;
268     break;
269     case 'c':
270     dump_coordinates=1;
271     break;
272     #ifdef HAVE_POSTGRESQL
273     case 'd':
274     dbstr=optarg;
275     break;
276     #endif
277     case 'e':
278     end=atoi(optarg);
279     break;
280     case 'h':
281     usage(stdout);
282     break;
283     case 'm':
284     optarg_cp=g_strdup(optarg);
285     pos=0;
286     i=0;
287     attr_name=g_strdup(optarg);
288     attr_value=g_strdup(optarg);
289     while (i < 9 && attr_from_line(optarg_cp, NULL, &pos, attr_value, attr_name)) {
290     attrs[i]=attr_new_from_text(attr_name,attr_value);
291     if (attrs[i]) {
292     i++;
293     } else {
294     fprintf(stderr,"Failed to convert %s=%s to attribute\n",attr_name,attr_value);
295     }
296     attr_value=g_strdup(optarg);
297     }
298     attrs[i++]=NULL;
299     g_free(attr_value);
300     g_free(optarg_cp);
301     handle=map_new(NULL, attrs);
302     if (! handle) {
303     fprintf(stderr,"Failed to create map from attributes\n");
304     exit(1);
305     }
306     map_handles=g_list_append(map_handles,handle);
307     break;
308     case 'n':
309     fprintf(stderr,"I will IGNORE unknown types\n");
310     ignore_unkown=1;
311     break;
312     case 'k':
313     fprintf(stderr,"I will KEEP tmp files\n");
314     keep_tmpfiles=1;
315     break;
316     case 'p':
317     add_plugin(optarg);
318     break;
319     case 's':
320     start=atoi(optarg);
321     break;
322     case 'w':
323     dedupe_ways_hash=g_hash_table_new(NULL, NULL);
324     break;
325     case 'i':
326     input_file = fopen( optarg, "r" );
327     if ( input_file == NULL )
328     {
329     fprintf( stderr, "\nInput file (%s) not found\n", optarg );
330     exit( -1 );
331     }
332     break;
333     case 'r':
334     rule_file = fopen( optarg, "r" );
335     if ( rule_file == NULL )
336     {
337     fprintf( stderr, "\nRule file (%s) not found\n", optarg );
338     exit( -1 );
339     }
340     break;
341     case 'u':
342     url=optarg;
343     break;
344     #ifdef HAVE_ZLIB
345     case 'z':
346     compression_level=atoi(optarg);
347     break;
348     #endif
349     case '?':
350     usage(stderr);
351     break;
352     default:
353     fprintf(stderr,"c=%d\n", c);
354     }
355    
356     }
357     if (optind != argc-(output == 1 ? 0:1))
358     usage(stderr);
359     result=argv[optind];
360    
361     // initialize plugins and OSM mappings
362     maptool_init(rule_file);
363     if (protobufdb_operation) {
364     osm_protobufdb_load(input_file, protobufdb);
365     return 0;
366     }
367    
368     // input from an OSM file
369     if (input == 0) {
370     if (start == 1) {
371     unlink("coords.tmp");
372     if (process_ways)
373     ways=tempfile(suffix,"ways",1);
374     if (process_nodes)
375     nodes=tempfile(suffix,"nodes",1);
376     if (process_ways && process_nodes)
377     turn_restrictions=tempfile(suffix,"turn_restrictions",1);
378     if (process_relations)
379     boundaries=tempfile(suffix,"boundaries",1);
380     phase=1;
381     fprintf(stderr,"PROGRESS: Phase 1: collecting data\n");
382     #ifdef HAVE_POSTGRESQL
383     if (dbstr)
384     map_collect_data_osm_db(dbstr,ways,nodes,turn_restrictions,boundaries);
385     else
386     #endif
387     if (map_handles) {
388     GList *l;
389     phase1_map(map_handles,ways,nodes);
390     l=map_handles;
391     while (l) {
392     map_destroy(l->data);
393     l=g_list_next(l);
394     }
395     }
396     else if (protobuf)
397     map_collect_data_osm_protobuf(input_file,ways,nodes,turn_restrictions,boundaries);
398     else
399     map_collect_data_osm(input_file,ways,nodes,turn_restrictions,boundaries);
400     if (slices) {
401     fprintf(stderr,"%d slices\n",slices);
402     flush_nodes(1);
403     for (i = slices-2 ; i>=0 ; i--) {
404     fprintf(stderr, "slice %d of %d\n",slices-i-1,slices-1);
405     load_buffer("coords.tmp",&node_buffer, i*slice_size, slice_size);
406     resolve_ways(ways, NULL);
407     save_buffer("coords.tmp",&node_buffer, i*slice_size);
408     }
409     } else
410     save_buffer("coords.tmp",&node_buffer, 0);
411     if (ways)
412     fclose(ways);
413     if (nodes)
414     fclose(nodes);
415     if (turn_restrictions)
416     fclose(turn_restrictions);
417     if (boundaries)
418     fclose(boundaries);
419     }
420     if (!slices) {
421     if (end == 1 || dump_coordinates)
422     flush_nodes(1);
423     else
424     slices++;
425     }
426     if (end == 1)
427     exit(0);
428     if (start == 2) {
429     load_buffer("coords.tmp",&node_buffer,0, slice_size);
430     }
431     if (start <= 2) {
432     if (process_ways) {
433     ways=tempfile(suffix,"ways",0);
434     phase=2;
435     fprintf(stderr,"PROGRESS: Phase 2: finding intersections\n");
436     for (i = 0 ; i < slices ; i++) {
437     int final=(i >= slices-1);
438     ways_split=tempfile(suffix,"ways_split",1);
439     ways_split_index=final ? tempfile(suffix,"ways_split_index",1) : NULL;
440     graph=tempfile(suffix,"graph",1);
441     coastline=tempfile(suffix,"coastline",1);
442     if (i)
443     load_buffer("coords.tmp",&node_buffer, i*slice_size, slice_size);
444     map_find_intersections(ways,ways_split,ways_split_index,graph,coastline,final);
445     fclose(ways_split);
446     if (ways_split_index)
447     fclose(ways_split_index);
448     fclose(ways);
449     fclose(graph);
450     fclose(coastline);
451     if (! final) {
452     tempfile_rename(suffix,"ways_split","ways_to_resolve");
453     ways=tempfile(suffix,"ways_to_resolve",0);
454     }
455     }
456     if(!keep_tmpfiles)
457     tempfile_unlink(suffix,"ways");
458     tempfile_unlink(suffix,"ways_to_resolve");
459     } else
460     fprintf(stderr,"PROGRESS: Skipping Phase 2\n");
461     }
462     free(node_buffer.base);
463     node_buffer.base=NULL;
464     node_buffer.malloced=0;
465     node_buffer.size=0;
466     if (end == 2)
467     exit(0);
468     } else {
469     ways_split=tempfile(suffix,"ways_split",0);
470     process_binfile(stdin, ways_split);
471     fclose(ways_split);
472     }
473    
474     #if 1
475     coastline=tempfile(suffix,"coastline",0);
476     if (coastline) {
477     ways_split=tempfile(suffix,"ways_split",2);
478     fprintf(stderr,"coastline=%p\n",coastline);
479     process_coastlines(coastline, ways_split);
480     fclose(ways_split);
481     fclose(coastline);
482     }
483     #endif
484     if (start <= 3) {
485     fprintf(stderr,"PROGRESS: Phase 3: sorting countries, generating turn restrictions\n");
486     sort_countries(keep_tmpfiles);
487     if (process_relations) {
488     #if 0
489     boundaries=tempfile(suffix,"boundaries",0);
490     ways_split=tempfile(suffix,"ways_split",0);
491     process_boundaries(boundaries, ways_split);
492     fclose(boundaries);
493     fclose(ways_split);
494     exit(0);
495     #endif
496     turn_restrictions=tempfile(suffix,"turn_restrictions",0);
497     if (turn_restrictions) {
498     relations=tempfile(suffix,"relations",1);
499     coords=fopen("coords.tmp","rb");
500     ways_split=tempfile(suffix,"ways_split",0);
501     ways_split_index=tempfile(suffix,"ways_split_index",0);
502     process_turn_restrictions(turn_restrictions,coords,ways_split,ways_split_index,relations);
503     fclose(ways_split_index);
504     fclose(ways_split);
505     fclose(coords);
506     fclose(relations);
507     fclose(turn_restrictions);
508     if(!keep_tmpfiles)
509     tempfile_unlink(suffix,"turn_restrictions");
510     }
511     }
512     if(!keep_tmpfiles)
513     tempfile_unlink(suffix,"ways_split_index");
514     }
515     if (end == 3)
516     exit(0);
517     if (output == 1) {
518     fprintf(stderr,"PROGRESS: Phase 4: dumping\n");
519     if (process_nodes) {
520     nodes=tempfile(suffix,"nodes",0);
521     if (nodes) {
522     dump(nodes);
523     fclose(nodes);
524     }
525     }
526     if (process_ways) {
527     ways_split=tempfile(suffix,"ways_split",0);
528     if (ways_split) {
529     dump(ways_split);
530     fclose(ways_split);
531     }
532     }
533     if (process_relations) {
534     relations=tempfile(suffix,"relations",0);
535     fprintf(stderr,"Relations=%p\n",relations);
536     if (relations) {
537     dump(relations);
538     fclose(relations);
539     }
540     }
541     exit(0);
542     }
543     for (i = suffix_start ; i < suffix_count ; i++) {
544     suffix=suffixes[i];
545     if (start <= 4) {
546     phase=3;
547     if (i == suffix_start) {
548     zip_info=zip_new();
549     zip_set_zip64(zip_info, zip64);
550     zip_set_timestamp(zip_info, timestamp);
551     }
552     zipnum=zip_get_zipnum(zip_info);
553     fprintf(stderr,"PROGRESS: Phase 4: generating tiles %s\n",suffix);
554     tilesdir=tempfile(suffix,"tilesdir",1);
555     if (!strcmp(suffix,r)) { /* Makes compiler happy due to bug 35903 in gcc */
556     ch_generate_tiles(suffixes[0],suffix,tilesdir,zip_info);
557     } else {
558     for (f = 0 ; f < 3 ; f++)
559     files[f]=NULL;
560     if (process_relations)
561     files[0]=tempfile(suffix,"relations",0);
562     if (process_ways)
563     files[1]=tempfile(suffix,"ways_split",0);
564     if (process_nodes)
565     files[2]=tempfile(suffix,"nodes",0);
566     phase4(files,3,0,suffix,tilesdir,zip_info);
567     for (f = 0 ; f < 3 ; f++) {
568     if (files[f])
569     fclose(files[f]);
570     }
571     }
572     fclose(tilesdir);
573     zip_set_zipnum(zip_info,zipnum);
574     }
575     if (end == 4)
576     exit(0);
577     if (zip_info) {
578     zip_destroy(zip_info);
579     zip_info=NULL;
580     }
581     if (start <= 5) {
582     phase=4;
583     fprintf(stderr,"PROGRESS: Phase 5: assembling map %s\n",suffix);
584     if (i == suffix_start) {
585     char *zipdir=tempfile_name("zipdir","");
586     char *zipindex=tempfile_name("index","");
587     zip_info=zip_new();
588     zip_set_zip64(zip_info, zip64);
589     zip_set_timestamp(zip_info, timestamp);
590     zip_set_maxnamelen(zip_info, 14+strlen(suffixes[0]));
591     zip_set_compression_level(zip_info, compression_level);
592     if (md5file)
593     zip_set_md5(zip_info, 1);
594     zip_open(zip_info, result, zipdir, zipindex);
595     if (url) {
596     map_information_attrs[1].type=attr_url;
597     map_information_attrs[1].u.str=url;
598     }
599     index_init(zip_info, 1);
600     }
601     if (!strcmp(suffix,r)) { /* Makes compiler happy due to bug 35903 in gcc */
602     ch_assemble_map(suffixes[0],suffix,zip_info);
603     } else {
604     for (f = 0 ; f < 3 ; f++) {
605     files[f]=NULL;
606     references[f]=NULL;
607     }
608     if (process_relations)
609     files[0]=tempfile(suffix,"relations",0);
610     if (process_ways) {
611     files[1]=tempfile(suffix,"ways_split",0);
612     references[1]=tempfile(suffix,"ways_split_ref",1);
613     }
614     if (process_nodes)
615     files[2]=tempfile(suffix,"nodes",0);
616     fprintf(stderr,"Slice %d\n",i);
617    
618     phase5(files,references,3,0,suffix,zip_info);
619     for (f = 0 ; f < 3 ; f++) {
620     if (files[f])
621     fclose(files[f]);
622     if (references[f])
623     fclose(references[f]);
624     }
625     }
626     if(!keep_tmpfiles) {
627     tempfile_unlink(suffix,"relations");
628     tempfile_unlink(suffix,"nodes");
629     tempfile_unlink(suffix,"ways_split");
630     tempfile_unlink(suffix,"ways_split_ref");
631     tempfile_unlink(suffix,"coastline");
632     tempfile_unlink(suffix,"turn_restrictions");
633     tempfile_unlink(suffix,"graph");
634     tempfile_unlink(suffix,"tilesdir");
635     tempfile_unlink(suffix,"boundaries");
636     unlink("coords.tmp");
637     }
638     if (i == suffix_count-1) {
639     unsigned char md5_data[16];
640     zipnum=zip_get_zipnum(zip_info);
641     add_aux_tiles("auxtiles.txt", zip_info);
642     write_countrydir(zip_info);
643     zip_set_zipnum(zip_info, zipnum);
644     write_aux_tiles(zip_info);
645     zip_write_index(zip_info);
646     zip_write_directory(zip_info);
647     zip_close(zip_info);
648     if (md5file && zip_get_md5(zip_info, md5_data)) {
649     FILE *md5=fopen(md5file,"w");
650     int i;
651     for (i = 0 ; i < 16 ; i++)
652     fprintf(md5,"%02x",md5_data[i]);
653     fprintf(md5,"\n");
654     fclose(md5);
655     }
656     if (!keep_tmpfiles) {
657     remove_countryfiles();
658     tempfile_unlink("index","");
659     tempfile_unlink("zipdir","");
660     }
661     }
662     }
663     }
664     return 0;
665     }

   
Visit the ZANavi Wiki