/[zanavi_public1]/navit/navit/maptool/osm_xml.c
ZANavi

Contents of /navit/navit/maptool/osm_xml.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 57 - (hide annotations) (download)
Sun Mar 19 16:46:08 2017 UTC (3 years, 10 months ago) by zoff99
File MIME type: text/plain
File size: 12088 byte(s)
updates
1 zoff99 8 /**
2 zoff99 37 * ZANavi, Zoff Android Navigation system.
3     * Copyright (C) 2011-2012 Zoff <zoff@zoff.cc>
4     *
5     * This program is free software; you can redistribute it and/or
6     * modify it under the terms of the GNU General Public License
7     * version 2 as published by the Free Software Foundation.
8     *
9     * This program is distributed in the hope that it will be useful,
10     * but WITHOUT ANY WARRANTY; without even the implied warranty of
11     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12     * GNU General Public License for more details.
13     *
14     * You should have received a copy of the GNU General Public License
15     * along with this program; if not, write to the
16     * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17     * Boston, MA 02110-1301, USA.
18     */
19    
20     /**
21 zoff99 8 * Navit, a modular navigation system.
22     * Copyright (C) 2005-2011 Navit Team
23     *
24     * This program is free software; you can redistribute it and/or
25     * modify it under the terms of the GNU General Public License
26     * version 2 as published by the Free Software Foundation.
27     *
28     * This program is distributed in the hope that it will be useful,
29     * but WITHOUT ANY WARRANTY; without even the implied warranty of
30     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31     * GNU General Public License for more details.
32     *
33     * You should have received a copy of the GNU General Public License
34     * along with this program; if not, write to the
35     * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
36     * Boston, MA 02110-1301, USA.
37     */
38     #include <string.h>
39     #include <stdlib.h>
40     #include <math.h>
41 zoff99 37 #include <time.h>
42 zoff99 8 #include <unistd.h>
43     #include "maptool.h"
44    
45 zoff99 37 int osm_xml_get_attribute(char *xml, char *attribute, char *buffer, int buffer_size)
46 zoff99 8 {
47 zoff99 37 int len = strlen(attribute);
48     char *pos, *i, s, attr[len + 2];
49 zoff99 8 strcpy(attr, attribute);
50 zoff99 37 strcpy(attr + len, "=");
51     pos = strstr(xml, attr);
52     if (!pos)
53 zoff99 8 return 0;
54 zoff99 37 pos += len + 1;
55     s = *pos++;
56     if (!s)
57 zoff99 8 return 0;
58 zoff99 37 i = strchr(pos, s);
59     if (!i)
60 zoff99 8 return 0;
61 zoff99 37 if (i - pos > buffer_size)
62     {
63     fprintf(stderr, "Buffer overflow %ld vs %d\n", (long) (i - pos), buffer_size);
64 zoff99 8 return 0;
65     }
66 zoff99 37 strncpy(buffer, pos, i - pos);
67     buffer[i - pos] = '\0';
68 zoff99 8 return 1;
69     }
70    
71 zoff99 37 static struct entity
72     {
73 zoff99 8 char *entity;
74     char c;
75 zoff99 37 } entities[] = { { "&quot;", '"' }, { "&apos;", '\'' }, { "&amp;", '&' }, { "&lt;", '<' }, { "&gt;", '>' }, { "&#34;", '"' }, { "&#39;", '\'' }, { "&#38;", '&' }, { "&#60;", '<' }, { "&#62;", '>' }, { "&#123;", '{' }, { "&#125;", '}' }, };
76 zoff99 8
77 zoff99 37 void osm_xml_decode_entities(char *buffer)
78 zoff99 8 {
79 zoff99 37 char *pos = buffer;
80     int i, len, found;
81 zoff99 8
82 zoff99 37 while ((pos = strchr(pos, '&')))
83     {
84     found = 0;
85     for (i = 0; i < sizeof(entities) / sizeof(struct entity); i++)
86     {
87     len = strlen(entities[i].entity);
88     if (!strncmp(pos, entities[i].entity, len))
89     {
90     *pos = entities[i].c;
91     memmove(pos + 1, pos + len, strlen(pos + len) + 1);
92     found = 1;
93 zoff99 8 break;
94     }
95     }
96     pos++;
97     }
98     }
99    
100 zoff99 37 static int parse_tag(char *p)
101 zoff99 8 {
102     char k_buffer[BUFFER_SIZE];
103     char v_buffer[BUFFER_SIZE];
104     if (!osm_xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE))
105 zoff99 37 {
106 zoff99 8 return 0;
107 zoff99 37 }
108 zoff99 8 if (!osm_xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE))
109 zoff99 37 {
110 zoff99 8 return 0;
111 zoff99 37 }
112 zoff99 8 osm_xml_decode_entities(v_buffer);
113     osm_add_tag(k_buffer, v_buffer);
114     return 1;
115     }
116    
117 zoff99 37 static int parse_node(char *p)
118 zoff99 8 {
119     char id_buffer[BUFFER_SIZE];
120     char lat_buffer[BUFFER_SIZE];
121     char lon_buffer[BUFFER_SIZE];
122     if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
123     return 0;
124     if (!osm_xml_get_attribute(p, "lat", lat_buffer, BUFFER_SIZE))
125     return 0;
126     if (!osm_xml_get_attribute(p, "lon", lon_buffer, BUFFER_SIZE))
127     return 0;
128     osm_add_node(atoll(id_buffer), atof(lat_buffer), atof(lon_buffer));
129     return 1;
130     }
131    
132 zoff99 37 static int parse_way(char *p)
133 zoff99 8 {
134     char id_buffer[BUFFER_SIZE];
135 zoff99 57
136 zoff99 8 if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
137 zoff99 57 {
138 zoff99 8 return 0;
139 zoff99 57 }
140    
141 zoff99 8 osm_add_way(atoll(id_buffer));
142     return 1;
143     }
144    
145 zoff99 37 static int parse_relation(char *p)
146 zoff99 8 {
147     char id_buffer[BUFFER_SIZE];
148     if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
149     return 0;
150     osm_add_relation(atoll(id_buffer));
151     return 1;
152     }
153    
154 zoff99 37 static int parse_member(char *p)
155 zoff99 8 {
156     char type_buffer[BUFFER_SIZE];
157     char ref_buffer[BUFFER_SIZE];
158     char role_buffer[BUFFER_SIZE];
159     int type;
160     if (!osm_xml_get_attribute(p, "type", type_buffer, BUFFER_SIZE))
161     return 0;
162     if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
163     return 0;
164     if (!osm_xml_get_attribute(p, "role", role_buffer, BUFFER_SIZE))
165     return 0;
166 zoff99 37 if (!strcmp(type_buffer, "node"))
167     type = 1;
168     else if (!strcmp(type_buffer, "way"))
169     type = 2;
170     else if (!strcmp(type_buffer, "relation"))
171     type = 3;
172     else
173     {
174 zoff99 36 //fprintf(stderr,"Unknown type %s\n",type_buffer);
175 zoff99 37 type = 0;
176 zoff99 8 }
177     osm_add_member(type, atoll(ref_buffer), role_buffer);
178 zoff99 37
179 zoff99 8 return 1;
180     }
181    
182 zoff99 37 static int parse_nd(char *p)
183 zoff99 8 {
184     char ref_buffer[BUFFER_SIZE];
185     if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
186 zoff99 37 {
187 zoff99 8 return 0;
188 zoff99 37 }
189 zoff99 8 osm_add_nd(atoll(ref_buffer));
190 zoff99 37
191 zoff99 8 return 1;
192     }
193    
194 zoff99 37 int map_collect_data_osm(FILE *in, struct maptool_osm *osm)
195 zoff99 8 {
196 zoff99 37 int size = 1024 * 5; // 5kb
197     // char buffer[size];
198     char *buffer_ptr;
199 zoff99 8 char *p;
200 zoff99 37
201     time_t start_tt, end_tt;
202     double diff_tt;
203     long long diff2_tt;
204     long long pos_in;
205     long long pos_in_local;
206     int _c = 0;
207     int _e = 5000000;
208     int first_rel = 1;
209     int first_way = 1;
210     int line_length = 0;
211    
212 zoff99 57 int percent_int_1 = 0;
213     int percent_int_2 = 0;
214     int percent_int_3 = 0;
215     int phase = 0; // 0:nodes 1:ways 2:relations
216    
217 zoff99 37 buffer_ptr = malloc(size);
218    
219     //sig_alrm(0);
220    
221     // reset
222     pos_in = 0;
223     pos_in_local = 0;
224     diff2_tt = 0;
225    
226     while (fgets(buffer_ptr, size, in))
227     {
228     // we just read "size" bytes from "in"
229    
230     line_length = strlen(buffer_ptr);
231    
232     pos_in = pos_in + line_length;
233     pos_in_local = pos_in_local + line_length;
234    
235     if (_c == 0)
236     {
237     time(&start_tt);
238     pos_in_local = 0;
239     }
240     _c++;
241    
242     p = strchr(buffer_ptr, '<');
243     if (!p)
244     {
245     //fprintf(stderr,"WARNING: wrong line %s\n", buffer_ptr);
246 zoff99 8 continue;
247     }
248 zoff99 37 if (!strncmp(p, "<?xml ", 6))
249     {
250     }
251     else if (!strncmp(p, "<osm ", 5))
252     {
253     }
254     else if (!strncmp(p, "<bound ", 7))
255     {
256     }
257     else if (!strncmp(p, "<node ", 6))
258     {
259 zoff99 8 if (!parse_node(p))
260 zoff99 36 {
261 zoff99 37 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
262 zoff99 36 }
263 zoff99 8 processed_nodes++;
264 zoff99 37 }
265     else if (!strncmp(p, "<tag ", 5))
266     {
267 zoff99 8 if (!parse_tag(p))
268 zoff99 36 {
269 zoff99 37 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
270 zoff99 36 }
271 zoff99 37 }
272     else if (!strncmp(p, "<way ", 5))
273     {
274     if (first_way == 1)
275     {
276     first_way = 0;
277    
278 zoff99 57 phase = 1;
279    
280 zoff99 37 flush_nodes(1, 0); // flush remaining nodes to "coords.tmp" from "osm_end_node"
281     free_buffer("dummy", &node_buffer[0]); // and free the memory
282    
283     sql_counter = 0;
284     sql_counter2 = 0;
285     sql_counter3 = 0;
286     sql_counter4 = 0;
287     sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
288     sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
289     sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
290     sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
291     sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
292     sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
293     sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
294     sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
295     sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
296     fprintf(stderr, "nodes processed:%lld\n", processed_nodes);
297    
298     if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
299     {
300     // reopen
301     sql_db_close();
302     sql_db_open();
303     sql_db_init(0);
304     }
305    
306     fprintf(stderr, "SQL: (create index 003) start\n");
307     sql_create_index003();
308     fprintf(stderr, "SQL: (create index 003) ready\n");
309    
310     if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
311     {
312     // reopen for indexes to be used
313     sql_db_close();
314     sql_db_open();
315     sql_db_init(0);
316     }
317    
318     fprintf(stderr, "SQL: (first way) COMMIT\n");
319    
320     }
321    
322 zoff99 8 if (!parse_way(p))
323 zoff99 36 {
324 zoff99 37 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
325 zoff99 36 }
326 zoff99 8 processed_ways++;
327 zoff99 37 }
328     else if (!strncmp(p, "<nd ", 4))
329     {
330 zoff99 8 if (!parse_nd(p))
331 zoff99 36 {
332 zoff99 37 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
333 zoff99 36 }
334 zoff99 37 }
335     else if (!strncmp(p, "<relation ", 10))
336     {
337     if (first_rel == 1)
338     {
339     first_rel = 0;
340    
341 zoff99 57 phase = 2;
342    
343 zoff99 37 sql_counter = 0;
344     sql_counter2 = 0;
345     sql_counter3 = 0;
346     sql_counter4 = 0;
347    
348     sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
349     sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
350     sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
351     sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
352     sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
353     sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
354     sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
355     sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
356     sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
357    
358     fprintf(stderr, "ways processed:%lld\n", processed_ways);
359    
360     if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
361     {
362     // reopen
363     sql_db_close();
364     sql_db_open();
365     sql_db_init(0);
366     }
367    
368     fprintf(stderr, "SQL: (create index 001) start\n");
369     sql_create_index001();
370     fprintf(stderr, "SQL: (create index 001) ready\n");
371    
372     if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
373     {
374     // reopen for indexes to be used
375     sql_db_close();
376     sql_db_open();
377     sql_db_init(0);
378     }
379    
380     fprintf(stderr, "SQL: (first relation) COMMIT\n");
381     }
382    
383 zoff99 8 if (!parse_relation(p))
384 zoff99 36 {
385 zoff99 37 if (verbose_mode)
386     fprintf(stderr, "WARNING: failed to parse %s\n", buffer_ptr);
387 zoff99 36 }
388 zoff99 8 processed_relations++;
389 zoff99 37 }
390     else if (!strncmp(p, "<member ", 8))
391     {
392 zoff99 8 if (!parse_member(p))
393 zoff99 36 {
394 zoff99 37 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
395 zoff99 36 }
396 zoff99 37 }
397     else if (!strncmp(p, "</node>", 7))
398     {
399 zoff99 36 osm_end_node(osm);
400 zoff99 37 }
401     else if (!strncmp(p, "</way>", 6))
402     {
403 zoff99 36 osm_end_way(osm);
404 zoff99 37 }
405     else if (!strncmp(p, "</relation>", 11))
406     {
407 zoff99 36 osm_end_relation(osm);
408 zoff99 8 }
409 zoff99 37 else if (!strncmp(p, "</osm>", 6))
410     {
411     }
412     else
413     {
414     // fprintf(stderr,"WARNING: unknown tag in %s\n", buffer_ptr);
415     }
416    
417     if (_c > _e)
418     {
419     _c = 0;
420    
421     time(&end_tt);
422     diff_tt = difftime(end_tt, start_tt);
423     diff2_tt = diff2_tt + (long) diff_tt;
424    
425     char outstring[200];
426     char outstring2[200];
427     convert_to_human_time(diff2_tt, outstring);
428     convert_to_human_bytes(pos_in, outstring2);
429 zoff99 57 fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s elapsed (%s read)\n", outstring, outstring2);
430 zoff99 37 convert_to_human_bytes((pos_in / diff2_tt), outstring2);
431 zoff99 57 // fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s/s read\n", outstring2);
432 zoff99 37
433     convert_to_human_bytes((pos_in_local / diff_tt), outstring2);
434 zoff99 57 fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA (local loop): %s/s read\n", outstring2);
435    
436     if (phase == 0)
437     {
438     if (processed_nodes_sum > 0)
439     {
440     percent_int_1 = (int)((processed_nodes * 100) / processed_nodes_sum);
441     }
442    
443     // fprintf_(stderr, "sum=%d nodes=%d\n", (int)processed_nodes_sum, (int)processed_nodes);
444     }
445     else if (phase == 1)
446     {
447     percent_int_1 = 100;
448     if (processed_ways_sum > 0)
449     {
450     percent_int_2 = (int)((processed_ways * 100) / processed_ways_sum);
451     }
452     }
453     else if (phase == 2)
454     {
455     percent_int_2 = 100;
456     if (processed_relations_sum > 0)
457     {
458     percent_int_3 = (int)((processed_relations * 100) / processed_relations_sum);
459     }
460     }
461    
462     fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: [%d% | %d% | %d%] read\n", percent_int_1, percent_int_2, percent_int_3);
463 zoff99 37 }
464 zoff99 8 }
465 zoff99 37
466     // just in case, commit all we got left over
467     //if (sql_counter > 0)
468     //{
469     sql_counter = 0;
470     sql_counter2 = 0;
471     sql_counter3 = 0;
472     sql_counter4 = 0;
473     sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
474     sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
475     sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
476     sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
477     sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
478     sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
479     sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
480     sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
481     sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
482     fprintf(stderr, "SQL: (final) COMMIT\n");
483     fprintf(stderr, "relations processed:%d\n", processed_relations);
484    
485     //}
486    
487     //sig_alrm(0);
488     //sig_alrm_end();
489    
490 zoff99 57 processed_relations_sum = processed_relations;
491     processed_ways_sum = processed_ways;
492     processed_nodes_sum = processed_nodes;
493    
494    
495 zoff99 37 free(buffer_ptr);
496    
497 zoff99 8 return 1;
498     }
499 zoff99 37

   
Visit the ZANavi Wiki