/[zanavi_public1]/navit/navit/maptool/osm_xml.c
ZANavi

Diff of /navit/navit/maptool/osm_xml.c

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

Revision 36 Revision 37
1/**
2 * ZANavi, Zoff Android Navigation system.
3 * Copyright (C) 2011-2012 Zoff <zoff@zoff.cc>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * version 2 as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
1/** 20/**
2 * Navit, a modular navigation system. 21 * Navit, a modular navigation system.
3 * Copyright (C) 2005-2011 Navit Team 22 * Copyright (C) 2005-2011 Navit Team
4 * 23 *
5 * This program is free software; you can redistribute it and/or 24 * This program is free software; you can redistribute it and/or
… …
17 * Boston, MA 02110-1301, USA. 36 * Boston, MA 02110-1301, USA.
18 */ 37 */
19#include <string.h> 38#include <string.h>
20#include <stdlib.h> 39#include <stdlib.h>
21#include <math.h> 40#include <math.h>
41#include <time.h>
22#include <unistd.h> 42#include <unistd.h>
23#include "maptool.h" 43#include "maptool.h"
24 44
25int
26osm_xml_get_attribute(char *xml, char *attribute, char *buffer, int buffer_size) 45int osm_xml_get_attribute(char *xml, char *attribute, char *buffer, int buffer_size)
27{ 46{
28 int len=strlen(attribute); 47 int len = strlen(attribute);
29 char *pos,*i,s,attr[len+2]; 48 char *pos, *i, s, attr[len + 2];
30 strcpy(attr, attribute); 49 strcpy(attr, attribute);
31 strcpy(attr+len, "="); 50 strcpy(attr + len, "=");
32 pos=strstr(xml, attr); 51 pos = strstr(xml, attr);
33 if (! pos) 52 if (!pos)
34 return 0; 53 return 0;
35 pos+=len+1; 54 pos += len + 1;
36 s=*pos++; 55 s = *pos++;
37 if (! s) 56 if (!s)
38 return 0; 57 return 0;
39 i=strchr(pos, s); 58 i = strchr(pos, s);
40 if (! i) 59 if (!i)
41 return 0; 60 return 0;
42 if (i - pos > buffer_size) { 61 if (i - pos > buffer_size)
62 {
43 fprintf(stderr,"Buffer overflow %ld vs %d\n", (long)(i-pos), buffer_size); 63 fprintf(stderr, "Buffer overflow %ld vs %d\n", (long) (i - pos), buffer_size);
44 return 0; 64 return 0;
45 } 65 }
46 strncpy(buffer, pos, i-pos); 66 strncpy(buffer, pos, i - pos);
47 buffer[i-pos]='\0'; 67 buffer[i - pos] = '\0';
48 return 1; 68 return 1;
49} 69}
50 70
51static struct entity { 71static struct entity
72{
52 char *entity; 73 char *entity;
53 char c; 74 char c;
54} entities[]= { 75} entities[] = { { "&quot;", '"' }, { "&apos;", '\'' }, { "&amp;", '&' }, { "&lt;", '<' }, { "&gt;", '>' }, { "&#34;", '"' }, { "&#39;", '\'' }, { "&#38;", '&' }, { "&#60;", '<' }, { "&#62;", '>' }, { "&#123;", '{' }, { "&#125;", '}' }, };
55 {"&quot;",'"'},
56 {"&apos;",'\''},
57 {"&amp;",'&'},
58 {"&lt;",'<'},
59 {"&gt;",'>'},
60 {"&#34;",'"'},
61 {"&#39;",'\''},
62 {"&#38;",'&'},
63 {"&#60;",'<'},
64 {"&#62;",'>'},
65 {"&#123;",'{'},
66 {"&#125;",'}'},
67};
68 76
69void
70osm_xml_decode_entities(char *buffer) 77void osm_xml_decode_entities(char *buffer)
71{ 78{
72 char *pos=buffer; 79 char *pos = buffer;
73 int i,len,found; 80 int i, len, found;
74 81
75 while ((pos=strchr(pos, '&'))) { 82 while ((pos = strchr(pos, '&')))
83 {
76 found=0; 84 found = 0;
77 for (i = 0 ; i < sizeof(entities)/sizeof(struct entity); i++) { 85 for (i = 0; i < sizeof(entities) / sizeof(struct entity); i++)
86 {
78 len=strlen(entities[i].entity); 87 len = strlen(entities[i].entity);
79 if (!strncmp(pos, entities[i].entity, len)) { 88 if (!strncmp(pos, entities[i].entity, len))
89 {
80 *pos=entities[i].c; 90 *pos = entities[i].c;
81 memmove(pos+1, pos+len, strlen(pos+len)+1); 91 memmove(pos + 1, pos + len, strlen(pos + len) + 1);
82 found=1; 92 found = 1;
83 break; 93 break;
84 } 94 }
85 } 95 }
86 pos++; 96 pos++;
87 } 97 }
88} 98}
89 99
90static int
91parse_tag(char *p) 100static int parse_tag(char *p)
92{ 101{
93 char k_buffer[BUFFER_SIZE]; 102 char k_buffer[BUFFER_SIZE];
94 char v_buffer[BUFFER_SIZE]; 103 char v_buffer[BUFFER_SIZE];
95 if (!osm_xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE)) 104 if (!osm_xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE))
105 {
96 return 0; 106 return 0;
107 }
97 if (!osm_xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE)) 108 if (!osm_xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE))
109 {
98 return 0; 110 return 0;
111 }
99 osm_xml_decode_entities(v_buffer); 112 osm_xml_decode_entities(v_buffer);
100 osm_add_tag(k_buffer, v_buffer); 113 osm_add_tag(k_buffer, v_buffer);
101 return 1; 114 return 1;
102} 115}
103 116
104
105static int
106parse_node(char *p) 117static int parse_node(char *p)
107{ 118{
108 char id_buffer[BUFFER_SIZE]; 119 char id_buffer[BUFFER_SIZE];
109 char lat_buffer[BUFFER_SIZE]; 120 char lat_buffer[BUFFER_SIZE];
110 char lon_buffer[BUFFER_SIZE]; 121 char lon_buffer[BUFFER_SIZE];
111 if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE)) 122 if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
… …
116 return 0; 127 return 0;
117 osm_add_node(atoll(id_buffer), atof(lat_buffer), atof(lon_buffer)); 128 osm_add_node(atoll(id_buffer), atof(lat_buffer), atof(lon_buffer));
118 return 1; 129 return 1;
119} 130}
120 131
121
122static int
123parse_way(char *p) 132static int parse_way(char *p)
124{ 133{
125 char id_buffer[BUFFER_SIZE]; 134 char id_buffer[BUFFER_SIZE];
126 if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE)) 135 if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
127 return 0; 136 return 0;
128 osm_add_way(atoll(id_buffer)); 137 osm_add_way(atoll(id_buffer));
129 return 1; 138 return 1;
130} 139}
131 140
132static int
133parse_relation(char *p) 141static int parse_relation(char *p)
134{ 142{
135 char id_buffer[BUFFER_SIZE]; 143 char id_buffer[BUFFER_SIZE];
136 if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE)) 144 if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
137 return 0; 145 return 0;
138 osm_add_relation(atoll(id_buffer)); 146 osm_add_relation(atoll(id_buffer));
139 return 1; 147 return 1;
140} 148}
141 149
142static int
143parse_member(char *p) 150static int parse_member(char *p)
144{ 151{
145 char type_buffer[BUFFER_SIZE]; 152 char type_buffer[BUFFER_SIZE];
146 char ref_buffer[BUFFER_SIZE]; 153 char ref_buffer[BUFFER_SIZE];
147 char role_buffer[BUFFER_SIZE]; 154 char role_buffer[BUFFER_SIZE];
148 int type; 155 int type;
… …
150 return 0; 157 return 0;
151 if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE)) 158 if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
152 return 0; 159 return 0;
153 if (!osm_xml_get_attribute(p, "role", role_buffer, BUFFER_SIZE)) 160 if (!osm_xml_get_attribute(p, "role", role_buffer, BUFFER_SIZE))
154 return 0; 161 return 0;
155 if (!strcmp(type_buffer,"node")) 162 if (!strcmp(type_buffer, "node"))
156 type=1; 163 type = 1;
157 else if (!strcmp(type_buffer,"way")) 164 else if (!strcmp(type_buffer, "way"))
158 type=2; 165 type = 2;
159 else if (!strcmp(type_buffer,"relation")) 166 else if (!strcmp(type_buffer, "relation"))
160 type=3; 167 type = 3;
161 else { 168 else
169 {
162 //fprintf(stderr,"Unknown type %s\n",type_buffer); 170 //fprintf(stderr,"Unknown type %s\n",type_buffer);
163 type=0; 171 type = 0;
164 } 172 }
165 osm_add_member(type, atoll(ref_buffer), role_buffer); 173 osm_add_member(type, atoll(ref_buffer), role_buffer);
166
167 return 1;
168}
169 174
170static int 175 return 1;
176}
177
171parse_nd(char *p) 178static int parse_nd(char *p)
172{ 179{
173 char ref_buffer[BUFFER_SIZE]; 180 char ref_buffer[BUFFER_SIZE];
174 if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE)) 181 if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
182 {
175 return 0; 183 return 0;
184 }
176 osm_add_nd(atoll(ref_buffer)); 185 osm_add_nd(atoll(ref_buffer));
177 return 1;
178}
179 186
180int 187 return 1;
188}
189
181map_collect_data_osm(FILE *in, struct maptool_osm *osm) 190int map_collect_data_osm(FILE *in, struct maptool_osm *osm)
182{ 191{
183 int size=BUFFER_SIZE; 192 int size = 1024 * 5; // 5kb
184 char buffer[size]; 193 // char buffer[size];
194 char *buffer_ptr;
185 char *p; 195 char *p;
196
197 time_t start_tt, end_tt;
198 double diff_tt;
199 long long diff2_tt;
200 long long pos_in;
201 long long pos_in_local;
202 int _c = 0;
203 int _e = 5000000;
204 int first_rel = 1;
205 int first_way = 1;
206 int line_length = 0;
207
208 buffer_ptr = malloc(size);
209
186 sig_alrm(0); 210 //sig_alrm(0);
211
212 // reset
213 pos_in = 0;
214 pos_in_local = 0;
215 diff2_tt = 0;
216
187 while (fgets(buffer, size, in)) { 217 while (fgets(buffer_ptr, size, in))
218 {
219 // we just read "size" bytes from "in"
220
221 line_length = strlen(buffer_ptr);
222
223 pos_in = pos_in + line_length;
224 pos_in_local = pos_in_local + line_length;
225
226 if (_c == 0)
227 {
228 time(&start_tt);
229 pos_in_local = 0;
230 }
231 _c++;
232
188 p=strchr(buffer,'<'); 233 p = strchr(buffer_ptr, '<');
189 if (! p) { 234 if (!p)
235 {
190 //fprintf(stderr,"WARNING: wrong line %s\n", buffer); 236 //fprintf(stderr,"WARNING: wrong line %s\n", buffer_ptr);
191 continue; 237 continue;
192 } 238 }
193 if (!strncmp(p, "<?xml ",6)) { 239 if (!strncmp(p, "<?xml ", 6))
240 {
241 }
194 } else if (!strncmp(p, "<osm ",5)) { 242 else if (!strncmp(p, "<osm ", 5))
243 {
244 }
195 } else if (!strncmp(p, "<bound ",7)) { 245 else if (!strncmp(p, "<bound ", 7))
246 {
247 }
196 } else if (!strncmp(p, "<node ",6)) { 248 else if (!strncmp(p, "<node ", 6))
249 {
197 if (!parse_node(p)) 250 if (!parse_node(p))
198 { 251 {
199 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer); 252 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
200 } 253 }
201 processed_nodes++; 254 processed_nodes++;
255 }
202 } else if (!strncmp(p, "<tag ",5)) { 256 else if (!strncmp(p, "<tag ", 5))
257 {
203 if (!parse_tag(p)) 258 if (!parse_tag(p))
204 { 259 {
205 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer); 260 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
261 }
206 } 262 }
207 } else if (!strncmp(p, "<way ",5)) { 263 else if (!strncmp(p, "<way ", 5))
264 {
265 if (first_way == 1)
266 {
267 first_way = 0;
268
269 flush_nodes(1, 0); // flush remaining nodes to "coords.tmp" from "osm_end_node"
270 free_buffer("dummy", &node_buffer[0]); // and free the memory
271
272 sql_counter = 0;
273 sql_counter2 = 0;
274 sql_counter3 = 0;
275 sql_counter4 = 0;
276 sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
277 sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
278 sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
279 sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
280 sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
281 sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
282 sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
283 sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
284 sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
285 fprintf(stderr, "nodes processed:%lld\n", processed_nodes);
286
287 if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
288 {
289 // reopen
290 sql_db_close();
291 sql_db_open();
292 sql_db_init(0);
293 }
294
295 fprintf(stderr, "SQL: (create index 003) start\n");
296 sql_create_index003();
297 fprintf(stderr, "SQL: (create index 003) ready\n");
298
299 if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
300 {
301 // reopen for indexes to be used
302 sql_db_close();
303 sql_db_open();
304 sql_db_init(0);
305 }
306
307 fprintf(stderr, "SQL: (first way) COMMIT\n");
308
309 }
310
208 if (!parse_way(p)) 311 if (!parse_way(p))
209 { 312 {
210 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer); 313 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
211 } 314 }
212 processed_ways++; 315 processed_ways++;
316 }
213 } else if (!strncmp(p, "<nd ",4)) { 317 else if (!strncmp(p, "<nd ", 4))
318 {
214 if (!parse_nd(p)) 319 if (!parse_nd(p))
215 { 320 {
216 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer); 321 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
322 }
217 } 323 }
218 } else if (!strncmp(p, "<relation ",10)) { 324 else if (!strncmp(p, "<relation ", 10))
325 {
326 if (first_rel == 1)
327 {
328 first_rel = 0;
329
330 sql_counter = 0;
331 sql_counter2 = 0;
332 sql_counter3 = 0;
333 sql_counter4 = 0;
334
335 sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
336 sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
337 sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
338 sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
339 sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
340 sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
341 sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
342 sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
343 sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
344
345 fprintf(stderr, "ways processed:%lld\n", processed_ways);
346
347 if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
348 {
349 // reopen
350 sql_db_close();
351 sql_db_open();
352 sql_db_init(0);
353 }
354
355 fprintf(stderr, "SQL: (create index 001) start\n");
356 sql_create_index001();
357 fprintf(stderr, "SQL: (create index 001) ready\n");
358
359 if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
360 {
361 // reopen for indexes to be used
362 sql_db_close();
363 sql_db_open();
364 sql_db_init(0);
365 }
366
367 fprintf(stderr, "SQL: (first relation) COMMIT\n");
368 }
369
219 if (!parse_relation(p)) 370 if (!parse_relation(p))
220 { 371 {
372 if (verbose_mode)
221 fprintf(stderr,"WARNING: failed to parse %s\n", buffer); 373 fprintf(stderr, "WARNING: failed to parse %s\n", buffer_ptr);
222 } 374 }
223 processed_relations++; 375 processed_relations++;
376 }
224 } else if (!strncmp(p, "<member ",8)) { 377 else if (!strncmp(p, "<member ", 8))
378 {
225 if (!parse_member(p)) 379 if (!parse_member(p))
226 { 380 {
227 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer); 381 //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
382 }
228 } 383 }
229 } else if (!strncmp(p, "</node>",7)) { 384 else if (!strncmp(p, "</node>", 7))
385 {
230 osm_end_node(osm); 386 osm_end_node(osm);
387 }
231 } else if (!strncmp(p, "</way>",6)) { 388 else if (!strncmp(p, "</way>", 6))
389 {
232 osm_end_way(osm); 390 osm_end_way(osm);
391 }
233 } else if (!strncmp(p, "</relation>",11)) { 392 else if (!strncmp(p, "</relation>", 11))
393 {
234 osm_end_relation(osm); 394 osm_end_relation(osm);
395 }
235 } else if (!strncmp(p, "</osm>",6)) { 396 else if (!strncmp(p, "</osm>", 6))
397 {
398 }
236 } else { 399 else
400 {
237 // fprintf(stderr,"WARNING: unknown tag in %s\n", buffer); 401 // fprintf(stderr,"WARNING: unknown tag in %s\n", buffer_ptr);
238 }
239 } 402 }
403
404 if (_c > _e)
405 {
406 _c = 0;
407
408 time(&end_tt);
409 diff_tt = difftime(end_tt, start_tt);
410 diff2_tt = diff2_tt + (long) diff_tt;
411
412 char outstring[200];
413 char outstring2[200];
414 convert_to_human_time(diff2_tt, outstring);
415 convert_to_human_bytes(pos_in, outstring2);
416 fprintf(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s elapsed (%s read)\n", outstring, outstring2);
417 convert_to_human_bytes((pos_in / diff2_tt), outstring2);
418 fprintf(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s/s read\n", outstring2);
419
420 convert_to_human_bytes((pos_in_local / diff_tt), outstring2);
421 fprintf(stderr, "-RUNTIME-LOOP-COLLECT-DATA (local loop): %s/s read\n", outstring2);
422 }
423 }
424
425 // just in case, commit all we got left over
426 //if (sql_counter > 0)
427 //{
428 sql_counter = 0;
429 sql_counter2 = 0;
430 sql_counter3 = 0;
431 sql_counter4 = 0;
432 sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
433 sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
434 sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
435 sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
436 sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
437 sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
438 sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
439 sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
440 sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
441 fprintf(stderr, "SQL: (final) COMMIT\n");
442 fprintf(stderr, "relations processed:%d\n", processed_relations);
443
444 //}
445
240 sig_alrm(0); 446 //sig_alrm(0);
241 sig_alrm_end(); 447 //sig_alrm_end();
448
449 free(buffer_ptr);
450
242 return 1; 451 return 1;
243} 452}
453
Legend:
Removed from v.36  
changed lines
  Added in v.37
   
Visit the ZANavi Wiki