/[zanavi_public1]/navit/navit/maptool/osm_xml.c
ZANavi

Contents of /navit/navit/maptool/osm_xml.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 57 - (show annotations) (download)
Sun Mar 19 16:46:08 2017 UTC (7 years ago) by zoff99
File MIME type: text/plain
File size: 12088 byte(s)
updates
1 /**
2 * ZANavi, Zoff Android Navigation system.
3 * Copyright (C) 2011-2012 Zoff <zoff@zoff.cc>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * version 2 as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20 /**
21 * Navit, a modular navigation system.
22 * Copyright (C) 2005-2011 Navit Team
23 *
24 * This program is free software; you can redistribute it and/or
25 * modify it under the terms of the GNU General Public License
26 * version 2 as published by the Free Software Foundation.
27 *
28 * This program is distributed in the hope that it will be useful,
29 * but WITHOUT ANY WARRANTY; without even the implied warranty of
30 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31 * GNU General Public License for more details.
32 *
33 * You should have received a copy of the GNU General Public License
34 * along with this program; if not, write to the
35 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
36 * Boston, MA 02110-1301, USA.
37 */
38 #include <string.h>
39 #include <stdlib.h>
40 #include <math.h>
41 #include <time.h>
42 #include <unistd.h>
43 #include "maptool.h"
44
45 int osm_xml_get_attribute(char *xml, char *attribute, char *buffer, int buffer_size)
46 {
47 int len = strlen(attribute);
48 char *pos, *i, s, attr[len + 2];
49 strcpy(attr, attribute);
50 strcpy(attr + len, "=");
51 pos = strstr(xml, attr);
52 if (!pos)
53 return 0;
54 pos += len + 1;
55 s = *pos++;
56 if (!s)
57 return 0;
58 i = strchr(pos, s);
59 if (!i)
60 return 0;
61 if (i - pos > buffer_size)
62 {
63 fprintf(stderr, "Buffer overflow %ld vs %d\n", (long) (i - pos), buffer_size);
64 return 0;
65 }
66 strncpy(buffer, pos, i - pos);
67 buffer[i - pos] = '\0';
68 return 1;
69 }
70
71 static struct entity
72 {
73 char *entity;
74 char c;
75 } entities[] = { { "&quot;", '"' }, { "&apos;", '\'' }, { "&amp;", '&' }, { "&lt;", '<' }, { "&gt;", '>' }, { "&#34;", '"' }, { "&#39;", '\'' }, { "&#38;", '&' }, { "&#60;", '<' }, { "&#62;", '>' }, { "&#123;", '{' }, { "&#125;", '}' }, };
76
77 void osm_xml_decode_entities(char *buffer)
78 {
79 char *pos = buffer;
80 int i, len, found;
81
82 while ((pos = strchr(pos, '&')))
83 {
84 found = 0;
85 for (i = 0; i < sizeof(entities) / sizeof(struct entity); i++)
86 {
87 len = strlen(entities[i].entity);
88 if (!strncmp(pos, entities[i].entity, len))
89 {
90 *pos = entities[i].c;
91 memmove(pos + 1, pos + len, strlen(pos + len) + 1);
92 found = 1;
93 break;
94 }
95 }
96 pos++;
97 }
98 }
99
100 static int parse_tag(char *p)
101 {
102 char k_buffer[BUFFER_SIZE];
103 char v_buffer[BUFFER_SIZE];
104 if (!osm_xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE))
105 {
106 return 0;
107 }
108 if (!osm_xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE))
109 {
110 return 0;
111 }
112 osm_xml_decode_entities(v_buffer);
113 osm_add_tag(k_buffer, v_buffer);
114 return 1;
115 }
116
117 static int parse_node(char *p)
118 {
119 char id_buffer[BUFFER_SIZE];
120 char lat_buffer[BUFFER_SIZE];
121 char lon_buffer[BUFFER_SIZE];
122 if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
123 return 0;
124 if (!osm_xml_get_attribute(p, "lat", lat_buffer, BUFFER_SIZE))
125 return 0;
126 if (!osm_xml_get_attribute(p, "lon", lon_buffer, BUFFER_SIZE))
127 return 0;
128 osm_add_node(atoll(id_buffer), atof(lat_buffer), atof(lon_buffer));
129 return 1;
130 }
131
132 static int parse_way(char *p)
133 {
134 char id_buffer[BUFFER_SIZE];
135
136 if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
137 {
138 return 0;
139 }
140
141 osm_add_way(atoll(id_buffer));
142 return 1;
143 }
144
145 static int parse_relation(char *p)
146 {
147 char id_buffer[BUFFER_SIZE];
148 if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
149 return 0;
150 osm_add_relation(atoll(id_buffer));
151 return 1;
152 }
153
154 static int parse_member(char *p)
155 {
156 char type_buffer[BUFFER_SIZE];
157 char ref_buffer[BUFFER_SIZE];
158 char role_buffer[BUFFER_SIZE];
159 int type;
160 if (!osm_xml_get_attribute(p, "type", type_buffer, BUFFER_SIZE))
161 return 0;
162 if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
163 return 0;
164 if (!osm_xml_get_attribute(p, "role", role_buffer, BUFFER_SIZE))
165 return 0;
166 if (!strcmp(type_buffer, "node"))
167 type = 1;
168 else if (!strcmp(type_buffer, "way"))
169 type = 2;
170 else if (!strcmp(type_buffer, "relation"))
171 type = 3;
172 else
173 {
174 //fprintf(stderr,"Unknown type %s\n",type_buffer);
175 type = 0;
176 }
177 osm_add_member(type, atoll(ref_buffer), role_buffer);
178
179 return 1;
180 }
181
182 static int parse_nd(char *p)
183 {
184 char ref_buffer[BUFFER_SIZE];
185 if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
186 {
187 return 0;
188 }
189 osm_add_nd(atoll(ref_buffer));
190
191 return 1;
192 }
193
194 int map_collect_data_osm(FILE *in, struct maptool_osm *osm)
195 {
196 int size = 1024 * 5; // 5kb
197 // char buffer[size];
198 char *buffer_ptr;
199 char *p;
200
201 time_t start_tt, end_tt;
202 double diff_tt;
203 long long diff2_tt;
204 long long pos_in;
205 long long pos_in_local;
206 int _c = 0;
207 int _e = 5000000;
208 int first_rel = 1;
209 int first_way = 1;
210 int line_length = 0;
211
212 int percent_int_1 = 0;
213 int percent_int_2 = 0;
214 int percent_int_3 = 0;
215 int phase = 0; // 0:nodes 1:ways 2:relations
216
217 buffer_ptr = malloc(size);
218
219 //sig_alrm(0);
220
221 // reset
222 pos_in = 0;
223 pos_in_local = 0;
224 diff2_tt = 0;
225
226 while (fgets(buffer_ptr, size, in))
227 {
228 // we just read "size" bytes from "in"
229
230 line_length = strlen(buffer_ptr);
231
232 pos_in = pos_in + line_length;
233 pos_in_local = pos_in_local + line_length;
234
235 if (_c == 0)
236 {
237 time(&start_tt);
238 pos_in_local = 0;
239 }
240 _c++;
241
242 p = strchr(buffer_ptr, '<');
243 if (!p)
244 {
245 //fprintf(stderr,"WARNING: wrong line %s\n", buffer_ptr);
246 continue;
247 }
248 if (!strncmp(p, "<?xml ", 6))
249 {
250 }
251 else if (!strncmp(p, "<osm ", 5))
252 {
253 }
254 else if (!strncmp(p, "<bound ", 7))
255 {
256 }
257 else if (!strncmp(p, "<node ", 6))
258 {
259 if (!parse_node(p))
260 {
261 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
262 }
263 processed_nodes++;
264 }
265 else if (!strncmp(p, "<tag ", 5))
266 {
267 if (!parse_tag(p))
268 {
269 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
270 }
271 }
272 else if (!strncmp(p, "<way ", 5))
273 {
274 if (first_way == 1)
275 {
276 first_way = 0;
277
278 phase = 1;
279
280 flush_nodes(1, 0); // flush remaining nodes to "coords.tmp" from "osm_end_node"
281 free_buffer("dummy", &node_buffer[0]); // and free the memory
282
283 sql_counter = 0;
284 sql_counter2 = 0;
285 sql_counter3 = 0;
286 sql_counter4 = 0;
287 sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
288 sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
289 sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
290 sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
291 sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
292 sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
293 sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
294 sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
295 sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
296 fprintf(stderr, "nodes processed:%lld\n", processed_nodes);
297
298 if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
299 {
300 // reopen
301 sql_db_close();
302 sql_db_open();
303 sql_db_init(0);
304 }
305
306 fprintf(stderr, "SQL: (create index 003) start\n");
307 sql_create_index003();
308 fprintf(stderr, "SQL: (create index 003) ready\n");
309
310 if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
311 {
312 // reopen for indexes to be used
313 sql_db_close();
314 sql_db_open();
315 sql_db_init(0);
316 }
317
318 fprintf(stderr, "SQL: (first way) COMMIT\n");
319
320 }
321
322 if (!parse_way(p))
323 {
324 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
325 }
326 processed_ways++;
327 }
328 else if (!strncmp(p, "<nd ", 4))
329 {
330 if (!parse_nd(p))
331 {
332 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
333 }
334 }
335 else if (!strncmp(p, "<relation ", 10))
336 {
337 if (first_rel == 1)
338 {
339 first_rel = 0;
340
341 phase = 2;
342
343 sql_counter = 0;
344 sql_counter2 = 0;
345 sql_counter3 = 0;
346 sql_counter4 = 0;
347
348 sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
349 sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
350 sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
351 sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
352 sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
353 sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
354 sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
355 sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
356 sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
357
358 fprintf(stderr, "ways processed:%lld\n", processed_ways);
359
360 if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
361 {
362 // reopen
363 sql_db_close();
364 sql_db_open();
365 sql_db_init(0);
366 }
367
368 fprintf(stderr, "SQL: (create index 001) start\n");
369 sql_create_index001();
370 fprintf(stderr, "SQL: (create index 001) ready\n");
371
372 if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
373 {
374 // reopen for indexes to be used
375 sql_db_close();
376 sql_db_open();
377 sql_db_init(0);
378 }
379
380 fprintf(stderr, "SQL: (first relation) COMMIT\n");
381 }
382
383 if (!parse_relation(p))
384 {
385 if (verbose_mode)
386 fprintf(stderr, "WARNING: failed to parse %s\n", buffer_ptr);
387 }
388 processed_relations++;
389 }
390 else if (!strncmp(p, "<member ", 8))
391 {
392 if (!parse_member(p))
393 {
394 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
395 }
396 }
397 else if (!strncmp(p, "</node>", 7))
398 {
399 osm_end_node(osm);
400 }
401 else if (!strncmp(p, "</way>", 6))
402 {
403 osm_end_way(osm);
404 }
405 else if (!strncmp(p, "</relation>", 11))
406 {
407 osm_end_relation(osm);
408 }
409 else if (!strncmp(p, "</osm>", 6))
410 {
411 }
412 else
413 {
414 // fprintf(stderr,"WARNING: unknown tag in %s\n", buffer_ptr);
415 }
416
417 if (_c > _e)
418 {
419 _c = 0;
420
421 time(&end_tt);
422 diff_tt = difftime(end_tt, start_tt);
423 diff2_tt = diff2_tt + (long) diff_tt;
424
425 char outstring[200];
426 char outstring2[200];
427 convert_to_human_time(diff2_tt, outstring);
428 convert_to_human_bytes(pos_in, outstring2);
429 fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s elapsed (%s read)\n", outstring, outstring2);
430 convert_to_human_bytes((pos_in / diff2_tt), outstring2);
431 // fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s/s read\n", outstring2);
432
433 convert_to_human_bytes((pos_in_local / diff_tt), outstring2);
434 fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA (local loop): %s/s read\n", outstring2);
435
436 if (phase == 0)
437 {
438 if (processed_nodes_sum > 0)
439 {
440 percent_int_1 = (int)((processed_nodes * 100) / processed_nodes_sum);
441 }
442
443 // fprintf_(stderr, "sum=%d nodes=%d\n", (int)processed_nodes_sum, (int)processed_nodes);
444 }
445 else if (phase == 1)
446 {
447 percent_int_1 = 100;
448 if (processed_ways_sum > 0)
449 {
450 percent_int_2 = (int)((processed_ways * 100) / processed_ways_sum);
451 }
452 }
453 else if (phase == 2)
454 {
455 percent_int_2 = 100;
456 if (processed_relations_sum > 0)
457 {
458 percent_int_3 = (int)((processed_relations * 100) / processed_relations_sum);
459 }
460 }
461
462 fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: [%d% | %d% | %d%] read\n", percent_int_1, percent_int_2, percent_int_3);
463 }
464 }
465
466 // just in case, commit all we got left over
467 //if (sql_counter > 0)
468 //{
469 sql_counter = 0;
470 sql_counter2 = 0;
471 sql_counter3 = 0;
472 sql_counter4 = 0;
473 sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
474 sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
475 sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
476 sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
477 sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
478 sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
479 sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
480 sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
481 sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
482 fprintf(stderr, "SQL: (final) COMMIT\n");
483 fprintf(stderr, "relations processed:%d\n", processed_relations);
484
485 //}
486
487 //sig_alrm(0);
488 //sig_alrm_end();
489
490 processed_relations_sum = processed_relations;
491 processed_ways_sum = processed_ways;
492 processed_nodes_sum = processed_nodes;
493
494
495 free(buffer_ptr);
496
497 return 1;
498 }
499

   
Visit the ZANavi Wiki