1 |
/**
|
2 |
* ZANavi, Zoff Android Navigation system.
|
3 |
* Copyright (C) 2011-2012 Zoff <zoff@zoff.cc>
|
4 |
*
|
5 |
* This program is free software; you can redistribute it and/or
|
6 |
* modify it under the terms of the GNU General Public License
|
7 |
* version 2 as published by the Free Software Foundation.
|
8 |
*
|
9 |
* This program is distributed in the hope that it will be useful,
|
10 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
12 |
* GNU General Public License for more details.
|
13 |
*
|
14 |
* You should have received a copy of the GNU General Public License
|
15 |
* along with this program; if not, write to the
|
16 |
* Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
|
17 |
* Boston, MA 02110-1301, USA.
|
18 |
*/
|
19 |
|
20 |
/**
|
21 |
* Navit, a modular navigation system.
|
22 |
* Copyright (C) 2005-2011 Navit Team
|
23 |
*
|
24 |
* This program is free software; you can redistribute it and/or
|
25 |
* modify it under the terms of the GNU General Public License
|
26 |
* version 2 as published by the Free Software Foundation.
|
27 |
*
|
28 |
* This program is distributed in the hope that it will be useful,
|
29 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
30 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
31 |
* GNU General Public License for more details.
|
32 |
*
|
33 |
* You should have received a copy of the GNU General Public License
|
34 |
* along with this program; if not, write to the
|
35 |
* Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
|
36 |
* Boston, MA 02110-1301, USA.
|
37 |
*/
|
38 |
#include <string.h>
|
39 |
#include <stdlib.h>
|
40 |
#include <math.h>
|
41 |
#include <time.h>
|
42 |
#include <unistd.h>
|
43 |
#include "maptool.h"
|
44 |
|
45 |
int osm_xml_get_attribute(char *xml, char *attribute, char *buffer, int buffer_size)
|
46 |
{
|
47 |
int len = strlen(attribute);
|
48 |
char *pos, *i, s, attr[len + 2];
|
49 |
strcpy(attr, attribute);
|
50 |
strcpy(attr + len, "=");
|
51 |
pos = strstr(xml, attr);
|
52 |
if (!pos)
|
53 |
return 0;
|
54 |
pos += len + 1;
|
55 |
s = *pos++;
|
56 |
if (!s)
|
57 |
return 0;
|
58 |
i = strchr(pos, s);
|
59 |
if (!i)
|
60 |
return 0;
|
61 |
if (i - pos > buffer_size)
|
62 |
{
|
63 |
fprintf(stderr, "Buffer overflow %ld vs %d\n", (long) (i - pos), buffer_size);
|
64 |
return 0;
|
65 |
}
|
66 |
strncpy(buffer, pos, i - pos);
|
67 |
buffer[i - pos] = '\0';
|
68 |
return 1;
|
69 |
}
|
70 |
|
71 |
static struct entity
|
72 |
{
|
73 |
char *entity;
|
74 |
char c;
|
75 |
} entities[] = { { """, '"' }, { "'", '\'' }, { "&", '&' }, { "<", '<' }, { ">", '>' }, { """, '"' }, { "'", '\'' }, { "&", '&' }, { "<", '<' }, { ">", '>' }, { "{", '{' }, { "}", '}' }, };
|
76 |
|
77 |
void osm_xml_decode_entities(char *buffer)
|
78 |
{
|
79 |
char *pos = buffer;
|
80 |
int i, len, found;
|
81 |
|
82 |
while ((pos = strchr(pos, '&')))
|
83 |
{
|
84 |
found = 0;
|
85 |
for (i = 0; i < sizeof(entities) / sizeof(struct entity); i++)
|
86 |
{
|
87 |
len = strlen(entities[i].entity);
|
88 |
if (!strncmp(pos, entities[i].entity, len))
|
89 |
{
|
90 |
*pos = entities[i].c;
|
91 |
memmove(pos + 1, pos + len, strlen(pos + len) + 1);
|
92 |
found = 1;
|
93 |
break;
|
94 |
}
|
95 |
}
|
96 |
pos++;
|
97 |
}
|
98 |
}
|
99 |
|
100 |
static int parse_tag(char *p)
|
101 |
{
|
102 |
char k_buffer[BUFFER_SIZE];
|
103 |
char v_buffer[BUFFER_SIZE];
|
104 |
if (!osm_xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE))
|
105 |
{
|
106 |
return 0;
|
107 |
}
|
108 |
if (!osm_xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE))
|
109 |
{
|
110 |
return 0;
|
111 |
}
|
112 |
osm_xml_decode_entities(v_buffer);
|
113 |
osm_add_tag(k_buffer, v_buffer);
|
114 |
return 1;
|
115 |
}
|
116 |
|
117 |
static int parse_node(char *p)
|
118 |
{
|
119 |
char id_buffer[BUFFER_SIZE];
|
120 |
char lat_buffer[BUFFER_SIZE];
|
121 |
char lon_buffer[BUFFER_SIZE];
|
122 |
if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
|
123 |
return 0;
|
124 |
if (!osm_xml_get_attribute(p, "lat", lat_buffer, BUFFER_SIZE))
|
125 |
return 0;
|
126 |
if (!osm_xml_get_attribute(p, "lon", lon_buffer, BUFFER_SIZE))
|
127 |
return 0;
|
128 |
osm_add_node(atoll(id_buffer), atof(lat_buffer), atof(lon_buffer));
|
129 |
return 1;
|
130 |
}
|
131 |
|
132 |
static int parse_way(char *p)
|
133 |
{
|
134 |
char id_buffer[BUFFER_SIZE];
|
135 |
|
136 |
if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
|
137 |
{
|
138 |
return 0;
|
139 |
}
|
140 |
|
141 |
osm_add_way(atoll(id_buffer));
|
142 |
return 1;
|
143 |
}
|
144 |
|
145 |
static int parse_relation(char *p)
|
146 |
{
|
147 |
char id_buffer[BUFFER_SIZE];
|
148 |
if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
|
149 |
return 0;
|
150 |
osm_add_relation(atoll(id_buffer));
|
151 |
return 1;
|
152 |
}
|
153 |
|
154 |
static int parse_member(char *p)
|
155 |
{
|
156 |
char type_buffer[BUFFER_SIZE];
|
157 |
char ref_buffer[BUFFER_SIZE];
|
158 |
char role_buffer[BUFFER_SIZE];
|
159 |
int type;
|
160 |
if (!osm_xml_get_attribute(p, "type", type_buffer, BUFFER_SIZE))
|
161 |
return 0;
|
162 |
if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
|
163 |
return 0;
|
164 |
if (!osm_xml_get_attribute(p, "role", role_buffer, BUFFER_SIZE))
|
165 |
return 0;
|
166 |
if (!strcmp(type_buffer, "node"))
|
167 |
type = 1;
|
168 |
else if (!strcmp(type_buffer, "way"))
|
169 |
type = 2;
|
170 |
else if (!strcmp(type_buffer, "relation"))
|
171 |
type = 3;
|
172 |
else
|
173 |
{
|
174 |
//fprintf(stderr,"Unknown type %s\n",type_buffer);
|
175 |
type = 0;
|
176 |
}
|
177 |
osm_add_member(type, atoll(ref_buffer), role_buffer);
|
178 |
|
179 |
return 1;
|
180 |
}
|
181 |
|
182 |
static int parse_nd(char *p)
|
183 |
{
|
184 |
char ref_buffer[BUFFER_SIZE];
|
185 |
if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
|
186 |
{
|
187 |
return 0;
|
188 |
}
|
189 |
osm_add_nd(atoll(ref_buffer));
|
190 |
|
191 |
return 1;
|
192 |
}
|
193 |
|
194 |
int map_collect_data_osm(FILE *in, struct maptool_osm *osm)
|
195 |
{
|
196 |
int size = 1024 * 5; // 5kb
|
197 |
// char buffer[size];
|
198 |
char *buffer_ptr;
|
199 |
char *p;
|
200 |
|
201 |
time_t start_tt, end_tt;
|
202 |
double diff_tt;
|
203 |
long long diff2_tt;
|
204 |
long long pos_in;
|
205 |
long long pos_in_local;
|
206 |
int _c = 0;
|
207 |
int _e = 5000000;
|
208 |
int first_rel = 1;
|
209 |
int first_way = 1;
|
210 |
int line_length = 0;
|
211 |
|
212 |
int percent_int_1 = 0;
|
213 |
int percent_int_2 = 0;
|
214 |
int percent_int_3 = 0;
|
215 |
int phase = 0; // 0:nodes 1:ways 2:relations
|
216 |
|
217 |
buffer_ptr = malloc(size);
|
218 |
|
219 |
//sig_alrm(0);
|
220 |
|
221 |
// reset
|
222 |
pos_in = 0;
|
223 |
pos_in_local = 0;
|
224 |
diff2_tt = 0;
|
225 |
|
226 |
while (fgets(buffer_ptr, size, in))
|
227 |
{
|
228 |
// we just read "size" bytes from "in"
|
229 |
|
230 |
line_length = strlen(buffer_ptr);
|
231 |
|
232 |
pos_in = pos_in + line_length;
|
233 |
pos_in_local = pos_in_local + line_length;
|
234 |
|
235 |
if (_c == 0)
|
236 |
{
|
237 |
time(&start_tt);
|
238 |
pos_in_local = 0;
|
239 |
}
|
240 |
_c++;
|
241 |
|
242 |
p = strchr(buffer_ptr, '<');
|
243 |
if (!p)
|
244 |
{
|
245 |
//fprintf(stderr,"WARNING: wrong line %s\n", buffer_ptr);
|
246 |
continue;
|
247 |
}
|
248 |
if (!strncmp(p, "<?xml ", 6))
|
249 |
{
|
250 |
}
|
251 |
else if (!strncmp(p, "<osm ", 5))
|
252 |
{
|
253 |
}
|
254 |
else if (!strncmp(p, "<bound ", 7))
|
255 |
{
|
256 |
}
|
257 |
else if (!strncmp(p, "<node ", 6))
|
258 |
{
|
259 |
if (!parse_node(p))
|
260 |
{
|
261 |
//fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
|
262 |
}
|
263 |
processed_nodes++;
|
264 |
}
|
265 |
else if (!strncmp(p, "<tag ", 5))
|
266 |
{
|
267 |
if (!parse_tag(p))
|
268 |
{
|
269 |
//fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
|
270 |
}
|
271 |
}
|
272 |
else if (!strncmp(p, "<way ", 5))
|
273 |
{
|
274 |
if (first_way == 1)
|
275 |
{
|
276 |
first_way = 0;
|
277 |
|
278 |
phase = 1;
|
279 |
|
280 |
flush_nodes(1, 0); // flush remaining nodes to "coords.tmp" from "osm_end_node"
|
281 |
free_buffer("dummy", &node_buffer[0]); // and free the memory
|
282 |
|
283 |
sql_counter = 0;
|
284 |
sql_counter2 = 0;
|
285 |
sql_counter3 = 0;
|
286 |
sql_counter4 = 0;
|
287 |
sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
|
288 |
sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
|
289 |
sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
|
290 |
sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
|
291 |
sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
|
292 |
sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
|
293 |
sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
|
294 |
sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
|
295 |
sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
|
296 |
fprintf(stderr, "nodes processed:%lld\n", processed_nodes);
|
297 |
|
298 |
if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
|
299 |
{
|
300 |
// reopen
|
301 |
sql_db_close();
|
302 |
sql_db_open();
|
303 |
sql_db_init(0);
|
304 |
}
|
305 |
|
306 |
fprintf(stderr, "SQL: (create index 003) start\n");
|
307 |
sql_create_index003();
|
308 |
fprintf(stderr, "SQL: (create index 003) ready\n");
|
309 |
|
310 |
if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
|
311 |
{
|
312 |
// reopen for indexes to be used
|
313 |
sql_db_close();
|
314 |
sql_db_open();
|
315 |
sql_db_init(0);
|
316 |
}
|
317 |
|
318 |
fprintf(stderr, "SQL: (first way) COMMIT\n");
|
319 |
|
320 |
}
|
321 |
|
322 |
if (!parse_way(p))
|
323 |
{
|
324 |
//fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
|
325 |
}
|
326 |
processed_ways++;
|
327 |
}
|
328 |
else if (!strncmp(p, "<nd ", 4))
|
329 |
{
|
330 |
if (!parse_nd(p))
|
331 |
{
|
332 |
//fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
|
333 |
}
|
334 |
}
|
335 |
else if (!strncmp(p, "<relation ", 10))
|
336 |
{
|
337 |
if (first_rel == 1)
|
338 |
{
|
339 |
first_rel = 0;
|
340 |
|
341 |
phase = 2;
|
342 |
|
343 |
sql_counter = 0;
|
344 |
sql_counter2 = 0;
|
345 |
sql_counter3 = 0;
|
346 |
sql_counter4 = 0;
|
347 |
|
348 |
sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
|
349 |
sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
|
350 |
sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
|
351 |
sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
|
352 |
sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
|
353 |
sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
|
354 |
sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
|
355 |
sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
|
356 |
sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
|
357 |
|
358 |
fprintf(stderr, "ways processed:%lld\n", processed_ways);
|
359 |
|
360 |
if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
|
361 |
{
|
362 |
// reopen
|
363 |
sql_db_close();
|
364 |
sql_db_open();
|
365 |
sql_db_init(0);
|
366 |
}
|
367 |
|
368 |
fprintf(stderr, "SQL: (create index 001) start\n");
|
369 |
sql_create_index001();
|
370 |
fprintf(stderr, "SQL: (create index 001) ready\n");
|
371 |
|
372 |
if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
|
373 |
{
|
374 |
// reopen for indexes to be used
|
375 |
sql_db_close();
|
376 |
sql_db_open();
|
377 |
sql_db_init(0);
|
378 |
}
|
379 |
|
380 |
fprintf(stderr, "SQL: (first relation) COMMIT\n");
|
381 |
}
|
382 |
|
383 |
if (!parse_relation(p))
|
384 |
{
|
385 |
if (verbose_mode)
|
386 |
fprintf(stderr, "WARNING: failed to parse %s\n", buffer_ptr);
|
387 |
}
|
388 |
processed_relations++;
|
389 |
}
|
390 |
else if (!strncmp(p, "<member ", 8))
|
391 |
{
|
392 |
if (!parse_member(p))
|
393 |
{
|
394 |
//fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
|
395 |
}
|
396 |
}
|
397 |
else if (!strncmp(p, "</node>", 7))
|
398 |
{
|
399 |
osm_end_node(osm);
|
400 |
}
|
401 |
else if (!strncmp(p, "</way>", 6))
|
402 |
{
|
403 |
osm_end_way(osm);
|
404 |
}
|
405 |
else if (!strncmp(p, "</relation>", 11))
|
406 |
{
|
407 |
osm_end_relation(osm);
|
408 |
}
|
409 |
else if (!strncmp(p, "</osm>", 6))
|
410 |
{
|
411 |
}
|
412 |
else
|
413 |
{
|
414 |
// fprintf(stderr,"WARNING: unknown tag in %s\n", buffer_ptr);
|
415 |
}
|
416 |
|
417 |
if (_c > _e)
|
418 |
{
|
419 |
_c = 0;
|
420 |
|
421 |
time(&end_tt);
|
422 |
diff_tt = difftime(end_tt, start_tt);
|
423 |
diff2_tt = diff2_tt + (long) diff_tt;
|
424 |
|
425 |
char outstring[200];
|
426 |
char outstring2[200];
|
427 |
convert_to_human_time(diff2_tt, outstring);
|
428 |
convert_to_human_bytes(pos_in, outstring2);
|
429 |
fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s elapsed (%s read)\n", outstring, outstring2);
|
430 |
convert_to_human_bytes((pos_in / diff2_tt), outstring2);
|
431 |
// fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s/s read\n", outstring2);
|
432 |
|
433 |
convert_to_human_bytes((pos_in_local / diff_tt), outstring2);
|
434 |
fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA (local loop): %s/s read\n", outstring2);
|
435 |
|
436 |
if (phase == 0)
|
437 |
{
|
438 |
if (processed_nodes_sum > 0)
|
439 |
{
|
440 |
percent_int_1 = (int)((processed_nodes * 100) / processed_nodes_sum);
|
441 |
}
|
442 |
|
443 |
// fprintf_(stderr, "sum=%d nodes=%d\n", (int)processed_nodes_sum, (int)processed_nodes);
|
444 |
}
|
445 |
else if (phase == 1)
|
446 |
{
|
447 |
percent_int_1 = 100;
|
448 |
if (processed_ways_sum > 0)
|
449 |
{
|
450 |
percent_int_2 = (int)((processed_ways * 100) / processed_ways_sum);
|
451 |
}
|
452 |
}
|
453 |
else if (phase == 2)
|
454 |
{
|
455 |
percent_int_2 = 100;
|
456 |
if (processed_relations_sum > 0)
|
457 |
{
|
458 |
percent_int_3 = (int)((processed_relations * 100) / processed_relations_sum);
|
459 |
}
|
460 |
}
|
461 |
|
462 |
fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: [%d% | %d% | %d%] read\n", percent_int_1, percent_int_2, percent_int_3);
|
463 |
}
|
464 |
}
|
465 |
|
466 |
// just in case, commit all we got left over
|
467 |
//if (sql_counter > 0)
|
468 |
//{
|
469 |
sql_counter = 0;
|
470 |
sql_counter2 = 0;
|
471 |
sql_counter3 = 0;
|
472 |
sql_counter4 = 0;
|
473 |
sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
|
474 |
sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
|
475 |
sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
|
476 |
sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
|
477 |
sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
|
478 |
sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
|
479 |
sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
|
480 |
sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
|
481 |
sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
|
482 |
fprintf(stderr, "SQL: (final) COMMIT\n");
|
483 |
fprintf(stderr, "relations processed:%d\n", processed_relations);
|
484 |
|
485 |
//}
|
486 |
|
487 |
//sig_alrm(0);
|
488 |
//sig_alrm_end();
|
489 |
|
490 |
processed_relations_sum = processed_relations;
|
491 |
processed_ways_sum = processed_ways;
|
492 |
processed_nodes_sum = processed_nodes;
|
493 |
|
494 |
|
495 |
free(buffer_ptr);
|
496 |
|
497 |
return 1;
|
498 |
}
|
499 |
|