1 | /** |
|
|
2 | * Navit, a modular navigation system. |
|
|
3 | * Copyright (C) 2005-2011 Navit Team |
|
|
4 | * |
|
|
5 | * This program is free software; you can redistribute it and/or |
|
|
6 | * modify it under the terms of the GNU General Public License |
|
|
7 | * version 2 as published by the Free Software Foundation. |
|
|
8 | * |
|
|
9 | * This program is distributed in the hope that it will be useful, |
|
|
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
|
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
|
12 | * GNU General Public License for more details. |
|
|
13 | * |
|
|
14 | * You should have received a copy of the GNU General Public License |
|
|
15 | * along with this program; if not, write to the |
|
|
16 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, |
|
|
17 | * Boston, MA 02110-1301, USA. |
|
|
18 | */ |
|
|
19 | #include <string.h> |
|
|
20 | #include <stdlib.h> |
|
|
21 | #include <math.h> |
|
|
22 | #include <unistd.h> |
|
|
23 | #include <time.h> |
|
|
24 | #include <zlib.h> |
|
|
25 | #include "maptool.h" |
|
|
26 | #include "debug.h" |
|
|
27 | #include "linguistics.h" |
|
|
28 | #include "file.h" |
|
|
29 | #include "generated-code/fileformat.pb-c.h" |
|
|
30 | #include "generated-code/osmformat.pb-c.h" |
|
|
31 | |
|
|
32 | static double latlon_scale=10000000.0; |
|
|
33 | |
|
|
34 | struct db_config { |
|
|
35 | int node_ids_per_file; |
|
|
36 | int node_ids_per_blob; |
|
|
37 | int node_ids_per_group; |
|
|
38 | int way_ids_per_file; |
|
|
39 | int way_ids_per_blob; |
|
|
40 | int way_ids_per_group; |
|
|
41 | int relation_ids_per_file; |
|
|
42 | int relation_ids_per_blob; |
|
|
43 | int relation_ids_per_group; |
|
|
44 | } db_config = { |
|
|
45 | 200000, 30000, 500, |
|
|
46 | 40000, 1000, 30, |
|
|
47 | 10000, 500, 20, |
|
|
48 | }; |
|
|
49 | |
|
|
50 | struct osm_protobufdb_context { |
|
|
51 | int current_file, current_block, active_block; |
|
|
52 | int in_node, in_way, in_relation; |
|
|
53 | OSMPBF__Node n; |
|
|
54 | OSMPBF__Way w; |
|
|
55 | OSMPBF__Relation r; |
|
|
56 | OSMPBF__Info i; |
|
|
57 | FILE *f; |
|
|
58 | OSMPBF__PrimitiveBlock *pb; |
|
|
59 | OSMPBF__PrimitiveGroup *pg; |
|
|
60 | GHashTable *string_hash; |
|
|
61 | OSMPBF__DenseInfo *di; |
|
|
62 | OSMPBF__DenseNodes *dn; |
|
|
63 | OSMPBF__StringTable *st; |
|
|
64 | } context; |
|
|
65 | |
|
|
66 | static int |
|
|
67 | osm_protobufdb_write_blob(OSMPBF__Blob *blob, FILE *out) |
|
|
68 | { |
|
|
69 | unsigned char lenb[4]; |
|
|
70 | int len,blen; |
|
|
71 | unsigned char *buffer; |
|
|
72 | OSMPBF__BlobHeader header=OSMPBF__BLOB_HEADER__INIT; |
|
|
73 | |
|
|
74 | |
|
|
75 | |
|
|
76 | blen=osmpbf__blob__get_packed_size(blob); |
|
|
77 | header.type="OSMData"; |
|
|
78 | header.datasize=blen; |
|
|
79 | len=osmpbf__blob_header__get_packed_size(&header); |
|
|
80 | buffer=alloca(len); |
|
|
81 | lenb[0]=len>>24; |
|
|
82 | lenb[1]=len>>16; |
|
|
83 | lenb[2]=len>>8; |
|
|
84 | lenb[3]=len; |
|
|
85 | osmpbf__blob_header__pack(&header, buffer); |
|
|
86 | if (fwrite(lenb, sizeof(lenb), 1, out) != 1) |
|
|
87 | return 0; |
|
|
88 | if (fwrite(buffer, len, 1, out) != 1) |
|
|
89 | return 0; |
|
|
90 | buffer=alloca(blen); |
|
|
91 | osmpbf__blob__pack(blob, buffer); |
|
|
92 | if (fwrite(buffer, blen, 1, out) != 1) |
|
|
93 | return 0; |
|
|
94 | return 1; |
|
|
95 | } |
|
|
96 | |
|
|
97 | #if 0 |
|
|
98 | void |
|
|
99 | dump_block(OSMPBF__PrimitiveBlock *pb) |
|
|
100 | { |
|
|
101 | int i,j; |
|
|
102 | printf("%d groups\n",pb->n_primitivegroup); |
|
|
103 | for (i = 0 ; i < pb->n_primitivegroup ; i++) { |
|
|
104 | printf("%d relations\n",pb->primitivegroup[i]->n_relations); |
|
|
105 | for (j = 0 ; j < pb->primitivegroup[i]->n_relations ; j++) { |
|
|
106 | printf("Info %d\n",pb->primitivegroup[i]->relations[j]->info->version); |
|
|
107 | } |
|
|
108 | } |
|
|
109 | } |
|
|
110 | #endif |
|
|
111 | |
|
|
112 | static int |
|
|
113 | osm_protobufdb_finish_block(struct osm_protobufdb_context *ctx) |
|
|
114 | { |
|
|
115 | OSMPBF__Blob *blob,empty_blob=OSMPBF__BLOB__INIT; |
|
|
116 | int len; |
|
|
117 | if (!ctx->pb) |
|
|
118 | return 0; |
|
|
119 | len=osmpbf__primitive_block__get_packed_size(ctx->pb); |
|
|
120 | |
|
|
121 | while (ctx->current_block < ctx->active_block) { |
|
|
122 | osm_protobufdb_write_blob(&empty_blob, ctx->f); |
|
|
123 | ctx->current_block++; |
|
|
124 | } |
|
|
125 | blob=malloc(sizeof(*blob)); |
|
|
126 | *blob=empty_blob; |
|
|
127 | blob->has_raw=1; |
|
|
128 | blob->has_raw_size=1; |
|
|
129 | blob->raw.data=malloc(len); |
|
|
130 | osmpbf__primitive_block__pack(ctx->pb, blob->raw.data); |
|
|
131 | blob->raw.len=len; |
|
|
132 | blob->raw_size=len; |
|
|
133 | osm_protobufdb_write_blob(blob, ctx->f); |
|
|
134 | osmpbf__blob__free_unpacked(blob, &protobuf_c_system_allocator); |
|
|
135 | osmpbf__primitive_block__free_unpacked(ctx->pb, &protobuf_c_system_allocator); |
|
|
136 | ctx->pb=NULL; |
|
|
137 | ctx->current_block++; |
|
|
138 | return 1; |
|
|
139 | } |
|
|
140 | |
|
|
141 | static int |
|
|
142 | osm_protobufdb_start_block(struct osm_protobufdb_context *ctx, int blocknum) |
|
|
143 | { |
|
|
144 | OSMPBF__PrimitiveBlock pb=OSMPBF__PRIMITIVE_BLOCK__INIT; |
|
|
145 | OSMPBF__StringTable st=OSMPBF__STRING_TABLE__INIT; |
|
|
146 | if (ctx->active_block == blocknum) |
|
|
147 | return 0; |
|
|
148 | osm_protobufdb_finish_block(ctx); |
|
|
149 | ctx->active_block=blocknum; |
|
|
150 | ctx->pb=malloc(sizeof(*ctx->pb)); |
|
|
151 | *ctx->pb=pb; |
|
|
152 | ctx->pb->stringtable=malloc(sizeof(*ctx->pb->stringtable)); |
|
|
153 | *ctx->pb->stringtable=st; |
|
|
154 | ctx->st=ctx->pb->stringtable; |
|
|
155 | return 1; |
|
|
156 | } |
|
|
157 | |
|
|
158 | static int |
|
|
159 | osm_protobufdb_start_group(struct osm_protobufdb_context *ctx, int groupnum) |
|
|
160 | { |
|
|
161 | OSMPBF__PrimitiveGroup pg=OSMPBF__PRIMITIVE_GROUP__INIT; |
|
|
162 | if (ctx->pb->n_primitivegroup <= groupnum) { |
|
|
163 | ctx->pb->primitivegroup=realloc(ctx->pb->primitivegroup, (groupnum+1)*sizeof(ctx->pb->primitivegroup[0])); |
|
|
164 | while (ctx->pb->n_primitivegroup <= groupnum) { |
|
|
165 | ctx->pb->primitivegroup[ctx->pb->n_primitivegroup]=malloc(sizeof(*context.pg)); |
|
|
166 | *ctx->pb->primitivegroup[ctx->pb->n_primitivegroup++]=pg; |
|
|
167 | } |
|
|
168 | g_hash_table_destroy(ctx->string_hash); |
|
|
169 | ctx->string_hash=g_hash_table_new(g_str_hash, g_str_equal); |
|
|
170 | } |
|
|
171 | ctx->pg=ctx->pb->primitivegroup[groupnum]; |
|
|
172 | if (!ctx->pg) { |
|
|
173 | ctx->pg=malloc(sizeof(*context.pg)); |
|
|
174 | *ctx->pg=pg; |
|
|
175 | ctx->pb->primitivegroup[groupnum]=ctx->pg; |
|
|
176 | } |
|
|
177 | return 1; |
|
|
178 | } |
|
|
179 | |
|
|
180 | static int |
|
|
181 | osm_protobufdb_start_densenode(struct osm_protobufdb_context *ctx) |
|
|
182 | { |
|
|
183 | OSMPBF__DenseInfo di=OSMPBF__DENSE_INFO__INIT; |
|
|
184 | OSMPBF__DenseNodes dn=OSMPBF__DENSE_NODES__INIT; |
|
|
185 | |
|
|
186 | if (!ctx->pg->dense) { |
|
|
187 | ctx->dn=malloc(sizeof(*context.dn)); |
|
|
188 | *ctx->dn=dn; |
|
|
189 | ctx->pg->dense=ctx->dn; |
|
|
190 | } else |
|
|
191 | ctx->dn=ctx->pg->dense; |
|
|
192 | if (!ctx->dn->denseinfo) { |
|
|
193 | ctx->di=malloc(sizeof(*context.di)); |
|
|
194 | *ctx->di=di; |
|
|
195 | ctx->dn->denseinfo=ctx->di; |
|
|
196 | } else |
|
|
197 | ctx->di=ctx->dn->denseinfo; |
|
|
198 | return 1; |
|
|
199 | } |
|
|
200 | |
|
|
201 | static void |
|
|
202 | osm_protobufdb_write_primitive_group(OSMPBF__PrimitiveGroup *pg, OSMPBF__PrimitiveBlock *pb) |
|
|
203 | { |
|
|
204 | pb->primitivegroup=realloc(pb->primitivegroup,(pb->n_primitivegroup+1)*sizeof(OSMPBF__PrimitiveGroup *)); |
|
|
205 | pb->primitivegroup[pb->n_primitivegroup++]=pg; |
|
|
206 | } |
|
|
207 | |
|
|
208 | |
|
|
209 | #define insert(struct, member, pos) {\ |
|
|
210 | int n=struct->n_##member; \ |
|
|
211 | int s=sizeof(struct->member[0]); \ |
|
|
212 | struct->member=realloc(struct->member, (n+1)*s); \ |
|
|
213 | memmove(&struct->member[n+1], &struct->member[n], (pos-n)*s); \ |
|
|
214 | memset(&struct->member[n], 0, s); \ |
|
|
215 | struct->n_##member++;\ |
|
|
216 | } |
|
|
217 | |
|
|
218 | static int |
|
|
219 | osm_protobufdb_insert_densenode(long long id, OSMPBF__Node *offset, OSMPBF__Info *offseti, OSMPBF__DenseNodes *dn) |
|
|
220 | { |
|
|
221 | int i,l,p; |
|
|
222 | memset(offset, 0, sizeof(*offset)); |
|
|
223 | offseti->timestamp=0; |
|
|
224 | offseti->changeset=0; |
|
|
225 | offseti->user_sid=0; |
|
|
226 | offseti->uid=0; |
|
|
227 | l=dn->n_id; |
|
|
228 | for (i = 0 ; i < l ; i++) { |
|
|
229 | offset->id+=dn->id[i]; |
|
|
230 | offset->lat+=dn->lat[i]; |
|
|
231 | offset->lon+=dn->lon[i]; |
|
|
232 | offseti->timestamp+=dn->denseinfo->timestamp[i]; |
|
|
233 | offseti->changeset+=dn->denseinfo->changeset[i]; |
|
|
234 | offseti->user_sid+=dn->denseinfo->user_sid[i]; |
|
|
235 | offseti->uid+=dn->denseinfo->uid[i]; |
|
|
236 | } |
|
|
237 | p=l; |
|
|
238 | insert(dn, id, p); |
|
|
239 | insert(dn, lat, p); |
|
|
240 | insert(dn, lon, p); |
|
|
241 | insert(dn->denseinfo, version, p); |
|
|
242 | insert(dn->denseinfo, timestamp, p); |
|
|
243 | insert(dn->denseinfo, changeset, p); |
|
|
244 | insert(dn->denseinfo, user_sid, p); |
|
|
245 | insert(dn->denseinfo, uid, p); |
|
|
246 | return p; |
|
|
247 | } |
|
|
248 | |
|
|
249 | static void |
|
|
250 | osm_protobufdb_modify_densenode(OSMPBF__Node *node, OSMPBF__Info *info, OSMPBF__Node *offset, OSMPBF__Info *offseti, int pos, OSMPBF__DenseNodes *dn) |
|
|
251 | { |
|
|
252 | int i; |
|
|
253 | if (pos+1 < dn->n_id) { |
|
|
254 | dn->id[pos+1]+=dn->id[pos]-node->id; |
|
|
255 | dn->lat[pos+1]+=dn->lat[pos]-node->lat; |
|
|
256 | dn->lon[pos+1]+=dn->lon[pos]-node->lon; |
|
|
257 | dn->denseinfo->timestamp[pos+1]+=dn->denseinfo->timestamp[pos]-info->timestamp; |
|
|
258 | dn->denseinfo->changeset[pos+1]+=dn->denseinfo->changeset[pos]-info->changeset; |
|
|
259 | dn->denseinfo->user_sid[pos+1]+=dn->denseinfo->user_sid[pos]-info->user_sid; |
|
|
260 | dn->denseinfo->uid[pos+1]+=dn->denseinfo->uid[pos]-info->uid; |
|
|
261 | } |
|
|
262 | dn->id[pos]=node->id-offset->id; |
|
|
263 | dn->lat[pos]=node->lat-offset->lat; |
|
|
264 | dn->lon[pos]=node->lon-offset->lon; |
|
|
265 | dn->keys_vals=realloc(dn->keys_vals, (dn->n_keys_vals+node->n_keys+node->n_vals+1)*sizeof(dn->keys_vals[0])); |
|
|
266 | for (i = 0 ; i < node->n_keys ; i++) { |
|
|
267 | dn->keys_vals[dn->n_keys_vals++]=node->keys[i]; |
|
|
268 | dn->keys_vals[dn->n_keys_vals++]=node->vals[i]; |
|
|
269 | } |
|
|
270 | dn->keys_vals[dn->n_keys_vals++]=0; |
|
|
271 | dn->denseinfo->version[pos]=info->version; |
|
|
272 | dn->denseinfo->timestamp[pos]=info->timestamp-offseti->timestamp; |
|
|
273 | dn->denseinfo->changeset[pos]=info->changeset-offseti->changeset; |
|
|
274 | dn->denseinfo->user_sid[pos]=info->user_sid-offseti->user_sid; |
|
|
275 | dn->denseinfo->uid[pos]=info->uid-offseti->uid; |
|
|
276 | } |
|
|
277 | |
|
|
278 | static int |
|
|
279 | osm_protobufdb_insert_node(long long id, OSMPBF__PrimitiveGroup *pg) |
|
|
280 | { |
|
|
281 | int l,p; |
|
|
282 | OSMPBF__Node node=OSMPBF__NODE__INIT; |
|
|
283 | l=pg->n_nodes; |
|
|
284 | p=l; |
|
|
285 | insert(pg, nodes, p); |
|
|
286 | pg->nodes[p]=malloc(sizeof(*pg->nodes[0])); |
|
|
287 | *pg->nodes[p]=node; |
|
|
288 | return p; |
|
|
289 | } |
|
|
290 | |
|
|
291 | static void |
|
|
292 | osm_protobufdb_modify_node(OSMPBF__Node *node, OSMPBF__Info *info, int pos, OSMPBF__PrimitiveGroup *pg) |
|
|
293 | { |
|
|
294 | OSMPBF__Node *n=pg->nodes[pos]; |
|
|
295 | OSMPBF__Info *old_info; |
|
|
296 | |
|
|
297 | if (n->keys) |
|
|
298 | free(n->keys); |
|
|
299 | if (n->vals) |
|
|
300 | free(n->vals); |
|
|
301 | old_info=n->info; |
|
|
302 | *n=*node; |
|
|
303 | if (!info) { |
|
|
304 | if (old_info) |
|
|
305 | osmpbf__info__free_unpacked(old_info, &protobuf_c_system_allocator); |
|
|
306 | n->info=NULL; |
|
|
307 | } else { |
|
|
308 | if (old_info) |
|
|
309 | n->info=old_info; |
|
|
310 | else |
|
|
311 | n->info=malloc(sizeof(*info)); |
|
|
312 | *n->info=*info; |
|
|
313 | } |
|
|
314 | |
|
|
315 | } |
|
|
316 | |
|
|
317 | static int |
|
|
318 | osm_protobufdb_insert_way(long long id, OSMPBF__PrimitiveGroup *pg) |
|
|
319 | { |
|
|
320 | int l,p; |
|
|
321 | OSMPBF__Way way=OSMPBF__WAY__INIT; |
|
|
322 | l=pg->n_ways; |
|
|
323 | p=l; |
|
|
324 | insert(pg, ways, p); |
|
|
325 | pg->ways[p]=malloc(sizeof(*pg->ways[0])); |
|
|
326 | *pg->ways[p]=way; |
|
|
327 | return p; |
|
|
328 | } |
|
|
329 | |
|
|
330 | static void |
|
|
331 | osm_protobufdb_modify_way(OSMPBF__Way *way, OSMPBF__Info *info, int pos, OSMPBF__PrimitiveGroup *pg) |
|
|
332 | { |
|
|
333 | OSMPBF__Way *w=pg->ways[pos]; |
|
|
334 | OSMPBF__Info *old_info; |
|
|
335 | int i; |
|
|
336 | long long ref=0; |
|
|
337 | |
|
|
338 | if (w->keys) |
|
|
339 | free(w->keys); |
|
|
340 | if (w->vals) |
|
|
341 | free(w->vals); |
|
|
342 | if (w->refs) |
|
|
343 | free(w->refs); |
|
|
344 | old_info=w->info; |
|
|
345 | *w=*way; |
|
|
346 | for (i = 0 ; i < w->n_refs ; i++) { |
|
|
347 | w->refs[i]-=ref; |
|
|
348 | ref+=w->refs[i]; |
|
|
349 | } |
|
|
350 | if (!info) { |
|
|
351 | if (old_info) |
|
|
352 | osmpbf__info__free_unpacked(old_info, &protobuf_c_system_allocator); |
|
|
353 | w->info=NULL; |
|
|
354 | } else { |
|
|
355 | if (old_info) |
|
|
356 | w->info=old_info; |
|
|
357 | else |
|
|
358 | w->info=malloc(sizeof(*info)); |
|
|
359 | *w->info=*info; |
|
|
360 | } |
|
|
361 | |
|
|
362 | } |
|
|
363 | |
|
|
364 | static int |
|
|
365 | osm_protobufdb_insert_relation(long long id, OSMPBF__PrimitiveGroup *pg) |
|
|
366 | { |
|
|
367 | int l,p; |
|
|
368 | OSMPBF__Relation relation=OSMPBF__RELATION__INIT; |
|
|
369 | l=pg->n_relations; |
|
|
370 | p=l; |
|
|
371 | insert(pg, relations, p); |
|
|
372 | pg->relations[p]=malloc(sizeof(*pg->relations[0])); |
|
|
373 | *pg->relations[p]=relation; |
|
|
374 | return p; |
|
|
375 | } |
|
|
376 | |
|
|
377 | static void |
|
|
378 | osm_protobufdb_modify_relation(OSMPBF__Relation *relation, OSMPBF__Info *info, int pos, OSMPBF__PrimitiveGroup *pg) |
|
|
379 | { |
|
|
380 | OSMPBF__Relation *r=pg->relations[pos]; |
|
|
381 | OSMPBF__Info *old_info; |
|
|
382 | int i; |
|
|
383 | long long ref=0; |
|
|
384 | |
|
|
385 | if (r->keys) |
|
|
386 | free(r->keys); |
|
|
387 | if (r->vals) |
|
|
388 | free(r->vals); |
|
|
389 | if (r->roles_sid) |
|
|
390 | free(r->roles_sid); |
|
|
391 | if (r->memids) |
|
|
392 | free(r->memids); |
|
|
393 | if (r->types) |
|
|
394 | free(r->types); |
|
|
395 | old_info=r->info; |
|
|
396 | *r=*relation; |
|
|
397 | for (i = 0 ; i < r->n_memids ; i++) { |
|
|
398 | r->memids[i]-=ref; |
|
|
399 | ref+=r->memids[i]; |
|
|
400 | } |
|
|
401 | if (!info) { |
|
|
402 | if (old_info) |
|
|
403 | osmpbf__info__free_unpacked(old_info, &protobuf_c_system_allocator); |
|
|
404 | r->info=NULL; |
|
|
405 | } else { |
|
|
406 | if (old_info) |
|
|
407 | r->info=old_info; |
|
|
408 | else |
|
|
409 | r->info=malloc(sizeof(*info)); |
|
|
410 | *r->info=*info; |
|
|
411 | } |
|
|
412 | |
|
|
413 | } |
|
|
414 | |
|
|
415 | static int |
|
|
416 | osm_protobufdb_string(struct osm_protobufdb_context *ctx, char *str) |
|
|
417 | { |
|
|
418 | char *strd; |
|
|
419 | OSMPBF__StringTable *st=ctx->st; |
|
|
420 | |
|
|
421 | gpointer value; |
|
|
422 | assert(ctx->string_hash != NULL); |
|
|
423 | if (g_hash_table_lookup_extended(ctx->string_hash, str, NULL, &value)) { |
|
|
424 | return (int)value; |
|
|
425 | } |
|
|
426 | if (!st->n_s) { |
|
|
427 | st->n_s++; |
|
|
428 | } |
|
|
429 | strd=strdup(str); |
|
|
430 | st->s=realloc(st->s, sizeof(st->s[0])*(st->n_s+1)); |
|
|
431 | if (st->n_s == 1) { |
|
|
432 | st->s[0].data=NULL; |
|
|
433 | st->s[0].len=0; |
|
|
434 | } |
|
|
435 | st->s[st->n_s].data=(unsigned char *)strd; |
|
|
436 | st->s[st->n_s].len=strlen(strd); |
|
|
437 | g_hash_table_insert(ctx->string_hash, strd, (gpointer)st->n_s); |
|
|
438 | return st->n_s++; |
|
|
439 | } |
|
|
440 | |
|
|
441 | |
|
|
442 | static int |
|
|
443 | osm_protobufdb_finish_file(struct osm_protobufdb_context *ctx) |
|
|
444 | { |
|
|
445 | osm_protobufdb_finish_block(ctx); |
|
|
446 | if (ctx->f) { |
|
|
447 | fclose(ctx->f); |
|
|
448 | ctx->f=NULL; |
|
|
449 | } |
|
|
450 | ctx->current_file=-1; |
|
|
451 | return 1; |
|
|
452 | } |
|
|
453 | |
|
|
454 | |
|
|
455 | static int |
|
|
456 | osm_protobufdb_start_file(struct osm_protobufdb_context *ctx, int type, int num) |
|
|
457 | { |
|
|
458 | char name[1024]; |
|
|
459 | if (ctx->current_file == num) |
|
|
460 | return 0; |
|
|
461 | osm_protobufdb_finish_file(ctx); |
|
|
462 | sprintf(name,"tst/%d-%08d",type,num); |
|
|
463 | ctx->f=fopen(name,"w"); |
|
|
464 | ctx->current_file=num; |
|
|
465 | ctx->current_block=0; |
|
|
466 | ctx->active_block=-1; |
|
|
467 | return 1; |
|
|
468 | } |
|
|
469 | |
|
|
470 | static void |
|
|
471 | test(void) |
|
|
472 | { |
|
|
473 | #if 0 |
|
|
474 | struct node n,o; |
|
|
475 | long long id=1; |
|
|
476 | long long lat=0; |
|
|
477 | long long lon=0; |
|
|
478 | long long timestamp=0; |
|
|
479 | long long changeset=0; |
|
|
480 | int version=1; |
|
|
481 | int user_sid=0; |
|
|
482 | int uid=0; |
|
|
483 | int p; |
|
|
484 | n.id=1; |
|
|
485 | #endif |
|
|
486 | #if 0 |
|
|
487 | OSMPBF__DenseInfo di=OSMPBF__DENSE_INFO__INIT; |
|
|
488 | OSMPBF__DenseNodes dn=OSMPBF__DENSE_NODES__INIT; |
|
|
489 | #endif |
|
|
490 | |
|
|
491 | context.current_file=-1; |
|
|
492 | |
|
|
493 | #if 0 |
|
|
494 | context.di=malloc(sizeof(*context.di)); |
|
|
495 | *context.di=di; |
|
|
496 | context.dn=malloc(sizeof(*context.dn)); |
|
|
497 | *context.dn=dn; |
|
|
498 | #endif |
|
|
499 | |
|
|
500 | #if 0 |
|
|
501 | di.n_version=1; |
|
|
502 | di.version=&version; |
|
|
503 | di.n_timestamp=1; |
|
|
504 | di.timestamp=×tamp; |
|
|
505 | di.n_changeset=1; |
|
|
506 | di.changeset=&changeset; |
|
|
507 | di.n_user_sid=1; |
|
|
508 | di.user_sid=&user_sid; |
|
|
509 | di.n_uid=1; |
|
|
510 | di.uid=&uid; |
|
|
511 | #endif |
|
|
512 | #if 0 |
|
|
513 | n.id=1; |
|
|
514 | n.lat=1; |
|
|
515 | n.lon=1; |
|
|
516 | n.timestamp=1; |
|
|
517 | n.changeset=1; |
|
|
518 | n.version=1; |
|
|
519 | n.user_sid=0; |
|
|
520 | n.uid=0; |
|
|
521 | p=osm_protobufdb_insert_densenode(&n.id, &o, &dn); |
|
|
522 | osm_protobufdb_modify(&n, &o, p, &dn); |
|
|
523 | p=osm_protobufdb_insert_densenode(&n.id, &o, &dn); |
|
|
524 | osm_protobufdb_modify(&n, &o, p, &dn); |
|
|
525 | #endif |
|
|
526 | |
|
|
527 | |
|
|
528 | #if 0 |
|
|
529 | dn.n_id=1; |
|
|
530 | dn.id=&id; |
|
|
531 | dn.n_lat=1; |
|
|
532 | dn.lat=⪫ |
|
|
533 | dn.n_lon=1; |
|
|
534 | dn.lon=&lon; |
|
|
535 | #endif |
|
|
536 | #if 0 |
|
|
537 | st.n_s=1; |
|
|
538 | data.data="Test"; |
|
|
539 | data.len=4; |
|
|
540 | st.s=&data; |
|
|
541 | #endif |
|
|
542 | } |
|
|
543 | |
|
|
544 | static void |
|
|
545 | finish(void) |
|
|
546 | { |
|
|
547 | osm_protobufdb_finish_file(&context); |
|
|
548 | #if 0 |
|
|
549 | osm_protobufdb_write_primitive_group(context.pg, context.pb); |
|
|
550 | osm_protobufdb_write_primitive_block(context.pb, context.blob); |
|
|
551 | osm_protobufdb_write_blob_to_file(); |
|
|
552 | #endif |
|
|
553 | } |
|
|
554 | |
|
|
555 | static long long |
|
|
556 | osm_protobufdb_timestamp(char *str) |
|
|
557 | { |
|
|
558 | struct tm tm; |
|
|
559 | int res=sscanf(str,"%d-%d-%dT%d:%d:%dZ",&tm.tm_year,&tm.tm_mon,&tm.tm_mday,&tm.tm_hour,&tm.tm_min,&tm.tm_sec); |
|
|
560 | if (res != 6) |
|
|
561 | return 0; |
|
|
562 | tm.tm_year-=1900; |
|
|
563 | tm.tm_mon-=1; |
|
|
564 | #if defined(HAVE_API_WIN32_BASE) || defined(ANDROID) |
|
|
565 | return 0; |
|
|
566 | #else |
|
|
567 | return timegm(&tm); |
|
|
568 | #endif |
|
|
569 | } |
|
|
570 | |
|
|
571 | static void |
|
|
572 | osm_protobufdb_parse_info(struct osm_protobufdb_context *ctx, char *str) |
|
|
573 | { |
|
|
574 | char version[1024]; |
|
|
575 | char changeset[1024]; |
|
|
576 | char user[1024]; |
|
|
577 | char uid[1024]; |
|
|
578 | char timestamp[1024]; |
|
|
579 | |
|
|
580 | OSMPBF__Info *i=&ctx->i, ii=OSMPBF__INFO__INIT; |
|
|
581 | *i=ii; |
|
|
582 | if (osm_xml_get_attribute(str, "version", version, sizeof(version))) { |
|
|
583 | i->version=atoi(version); |
|
|
584 | i->has_version=1; |
|
|
585 | } |
|
|
586 | if (osm_xml_get_attribute(str, "changeset", changeset, sizeof(changeset))) { |
|
|
587 | i->changeset=atoll(changeset); |
|
|
588 | i->has_changeset=1; |
|
|
589 | } |
|
|
590 | if (osm_xml_get_attribute(str, "user", user, sizeof(user))) { |
|
|
591 | osm_xml_decode_entities(user); |
|
|
592 | i->user_sid=osm_protobufdb_string(ctx, user); |
|
|
593 | i->has_user_sid=1; |
|
|
594 | } |
|
|
595 | if (osm_xml_get_attribute(str, "uid", uid, sizeof(uid))) { |
|
|
596 | i->uid=atoi(uid); |
|
|
597 | i->has_uid=1; |
|
|
598 | } |
|
|
599 | if (osm_xml_get_attribute(str, "timestamp", timestamp, sizeof(timestamp))) { |
|
|
600 | i->timestamp=osm_protobufdb_timestamp(timestamp); |
|
|
601 | i->has_timestamp=1; |
|
|
602 | } |
|
|
603 | } |
|
|
604 | |
|
|
605 | static int |
|
|
606 | osm_protobufdb_parse_node(struct osm_protobufdb_context *ctx, char *str) |
|
|
607 | { |
|
|
608 | char id[1024]; |
|
|
609 | char lat[1024]; |
|
|
610 | char lon[1024]; |
|
|
611 | OSMPBF__Node *n=&ctx->n, ni=OSMPBF__NODE__INIT; |
|
|
612 | |
|
|
613 | *n=ni; |
|
|
614 | if (!osm_xml_get_attribute(str, "id", id, sizeof(id))) |
|
|
615 | return 0; |
|
|
616 | if (!osm_xml_get_attribute(str, "lat", lat, sizeof(lat))) |
|
|
617 | return 0; |
|
|
618 | if (!osm_xml_get_attribute(str, "lon", lon, sizeof(lon))) |
|
|
619 | return 0; |
|
|
620 | n->id=atoll(id); |
|
|
621 | n->lat=atof(lat)*latlon_scale+0.5; |
|
|
622 | n->lon=atof(lon)*latlon_scale+0.5; |
|
|
623 | int file=n->id/db_config.node_ids_per_file; |
|
|
624 | int fileo=n->id%db_config.node_ids_per_file; |
|
|
625 | int blob=fileo/db_config.node_ids_per_blob; |
|
|
626 | int blobo=fileo%db_config.node_ids_per_blob; |
|
|
627 | int group=blobo/db_config.node_ids_per_group; |
|
|
628 | |
|
|
629 | osm_protobufdb_start_file(ctx, 1, file); |
|
|
630 | osm_protobufdb_start_block(ctx, blob); |
|
|
631 | osm_protobufdb_start_group(ctx, group); |
|
|
632 | osm_protobufdb_parse_info(ctx, str); |
|
|
633 | ctx->in_node=1; |
|
|
634 | return 1; |
|
|
635 | } |
|
|
636 | |
|
|
637 | static int |
|
|
638 | osm_protobufdb_end_node(struct osm_protobufdb_context *ctx) |
|
|
639 | { |
|
|
640 | int p; |
|
|
641 | #if 0 |
|
|
642 | OSMPBF__Node *n=&ctx->n,offset; |
|
|
643 | OSMPBF__Info *i=&ctx->i,offseti; |
|
|
644 | osm_protobufdb_start_densenode(ctx); |
|
|
645 | p=osm_protobufdb_insert_densenode(n->id, &offset, &offseti, context.dn); |
|
|
646 | osm_protobufdb_modify_densenode(n, i, &offset, &offseti, p, context.dn); |
|
|
647 | #else |
|
|
648 | p=osm_protobufdb_insert_node(ctx->n.id, ctx->pg); |
|
|
649 | osm_protobufdb_modify_node(&ctx->n, &ctx->i, p, ctx->pg); |
|
|
650 | #endif |
|
|
651 | ctx->in_node=0; |
|
|
652 | return 1; |
|
|
653 | } |
|
|
654 | |
|
|
655 | |
|
|
656 | static int |
|
|
657 | osm_protobufdb_parse_way(struct osm_protobufdb_context *ctx, char *str) |
|
|
658 | { |
|
|
659 | char id[1024]; |
|
|
660 | OSMPBF__Way *w=&ctx->w, wi=OSMPBF__WAY__INIT; |
|
|
661 | |
|
|
662 | *w=wi; |
|
|
663 | if (!osm_xml_get_attribute(str, "id", id, sizeof(id))) |
|
|
664 | return 0; |
|
|
665 | w->id=atoll(id); |
|
|
666 | int file=w->id/db_config.way_ids_per_file; |
|
|
667 | int fileo=w->id%db_config.way_ids_per_file; |
|
|
668 | int blob=fileo/db_config.way_ids_per_blob; |
|
|
669 | int blobo=fileo%db_config.way_ids_per_blob; |
|
|
670 | int group=blobo/db_config.way_ids_per_group; |
|
|
671 | |
|
|
672 | osm_protobufdb_start_file(ctx, 2, file); |
|
|
673 | osm_protobufdb_start_block(ctx, blob); |
|
|
674 | osm_protobufdb_start_group(ctx, group); |
|
|
675 | osm_protobufdb_parse_info(ctx, str); |
|
|
676 | ctx->in_way=1; |
|
|
677 | return 1; |
|
|
678 | } |
|
|
679 | |
|
|
680 | static int |
|
|
681 | osm_protobufdb_end_way(struct osm_protobufdb_context *ctx) |
|
|
682 | { |
|
|
683 | int p; |
|
|
684 | p=osm_protobufdb_insert_way(ctx->w.id, ctx->pg); |
|
|
685 | osm_protobufdb_modify_way(&ctx->w, &ctx->i, p, ctx->pg); |
|
|
686 | ctx->in_way=0; |
|
|
687 | return 1; |
|
|
688 | } |
|
|
689 | |
|
|
690 | static int |
|
|
691 | osm_protobufdb_parse_relation(struct osm_protobufdb_context *ctx, char *str) |
|
|
692 | { |
|
|
693 | char id[1024]; |
|
|
694 | OSMPBF__Relation *r=&ctx->r, ri=OSMPBF__RELATION__INIT; |
|
|
695 | |
|
|
696 | *r=ri; |
|
|
697 | if (!osm_xml_get_attribute(str, "id", id, sizeof(id))) |
|
|
698 | return 0; |
|
|
699 | r->id=atoll(id); |
|
|
700 | int file=r->id/db_config.relation_ids_per_file; |
|
|
701 | int fileo=r->id%db_config.relation_ids_per_file; |
|
|
702 | int blob=fileo/db_config.relation_ids_per_blob; |
|
|
703 | int blobo=fileo%db_config.relation_ids_per_blob; |
|
|
704 | int group=blobo/db_config.relation_ids_per_group; |
|
|
705 | |
|
|
706 | osm_protobufdb_start_file(ctx, 3, file); |
|
|
707 | osm_protobufdb_start_block(ctx, blob); |
|
|
708 | osm_protobufdb_start_group(ctx, group); |
|
|
709 | osm_protobufdb_parse_info(ctx, str); |
|
|
710 | ctx->in_relation=1; |
|
|
711 | return 1; |
|
|
712 | } |
|
|
713 | |
|
|
714 | static int |
|
|
715 | osm_protobufdb_end_relation(struct osm_protobufdb_context *ctx) |
|
|
716 | { |
|
|
717 | int p; |
|
|
718 | p=osm_protobufdb_insert_relation(ctx->r.id, ctx->pg); |
|
|
719 | osm_protobufdb_modify_relation(&ctx->r, &ctx->i, p, ctx->pg); |
|
|
720 | ctx->in_node=0; |
|
|
721 | return 1; |
|
|
722 | } |
|
|
723 | |
|
|
724 | static int |
|
|
725 | osm_protobufdb_parse_tag(struct osm_protobufdb_context *ctx, char *str) |
|
|
726 | { |
|
|
727 | OSMPBF__Node *n=&ctx->n; |
|
|
728 | OSMPBF__Way *w=&ctx->w; |
|
|
729 | OSMPBF__Relation *r=&ctx->r; |
|
|
730 | char k_buffer[BUFFER_SIZE]; |
|
|
731 | char v_buffer[BUFFER_SIZE]; |
|
|
732 | if (!osm_xml_get_attribute(str, "k", k_buffer, BUFFER_SIZE)) |
|
|
733 | return 0; |
|
|
734 | if (!osm_xml_get_attribute(str, "v", v_buffer, BUFFER_SIZE)) |
|
|
735 | return 0; |
|
|
736 | osm_xml_decode_entities(v_buffer); |
|
|
737 | if (ctx->in_node) { |
|
|
738 | n->keys=realloc(n->keys, (n->n_keys+1)*sizeof(n->keys[0])); |
|
|
739 | n->vals=realloc(n->vals, (n->n_vals+1)*sizeof(n->vals[0])); |
|
|
740 | n->keys[n->n_keys++]=osm_protobufdb_string(ctx, k_buffer); |
|
|
741 | n->vals[n->n_vals++]=osm_protobufdb_string(ctx, v_buffer); |
|
|
742 | } |
|
|
743 | if (ctx->in_way) { |
|
|
744 | w->keys=realloc(w->keys, (w->n_keys+1)*sizeof(w->keys[0])); |
|
|
745 | w->vals=realloc(w->vals, (w->n_vals+1)*sizeof(w->vals[0])); |
|
|
746 | w->keys[w->n_keys++]=osm_protobufdb_string(ctx, k_buffer); |
|
|
747 | w->vals[w->n_vals++]=osm_protobufdb_string(ctx, v_buffer); |
|
|
748 | } |
|
|
749 | if (ctx->in_relation) { |
|
|
750 | r->keys=realloc(r->keys, (r->n_keys+1)*sizeof(r->keys[0])); |
|
|
751 | r->vals=realloc(r->vals, (r->n_vals+1)*sizeof(r->vals[0])); |
|
|
752 | r->keys[r->n_keys++]=osm_protobufdb_string(ctx, k_buffer); |
|
|
753 | r->vals[r->n_vals++]=osm_protobufdb_string(ctx, v_buffer); |
|
|
754 | } |
|
|
755 | return 1; |
|
|
756 | } |
|
|
757 | |
|
|
758 | static int |
|
|
759 | osm_protobufdb_parse_nd(struct osm_protobufdb_context *ctx, char *str) |
|
|
760 | { |
|
|
761 | OSMPBF__Way *w=&ctx->w; |
|
|
762 | char ref_buffer[BUFFER_SIZE]; |
|
|
763 | if (!osm_xml_get_attribute(str, "ref", ref_buffer, BUFFER_SIZE)) |
|
|
764 | return 0; |
|
|
765 | if (ctx->in_way) { |
|
|
766 | w->refs=realloc(w->refs, (w->n_refs+1)*sizeof(w->refs[0])); |
|
|
767 | w->refs[w->n_refs++]=atoll(ref_buffer); |
|
|
768 | } |
|
|
769 | return 1; |
|
|
770 | } |
|
|
771 | |
|
|
772 | static int |
|
|
773 | osm_protobufdb_parse_member(struct osm_protobufdb_context *ctx, char *str) |
|
|
774 | { |
|
|
775 | OSMPBF__Relation *r=&ctx->r; |
|
|
776 | char type_buffer[BUFFER_SIZE]; |
|
|
777 | char ref_buffer[BUFFER_SIZE]; |
|
|
778 | char role_buffer[BUFFER_SIZE]; |
|
|
779 | int type=0; |
|
|
780 | if (!osm_xml_get_attribute(str, "type", type_buffer, BUFFER_SIZE)) |
|
|
781 | return 0; |
|
|
782 | if (!osm_xml_get_attribute(str, "ref", ref_buffer, BUFFER_SIZE)) |
|
|
783 | return 0; |
|
|
784 | if (!osm_xml_get_attribute(str, "role", role_buffer, BUFFER_SIZE)) |
|
|
785 | return 0; |
|
|
786 | if (!strcmp(type_buffer,"node")) |
|
|
787 | type=0; |
|
|
788 | else if (!strcmp(type_buffer,"way")) |
|
|
789 | type=1; |
|
|
790 | else if (!strcmp(type_buffer,"relation")) |
|
|
791 | type=2; |
|
|
792 | if (ctx->in_relation) { |
|
|
793 | r->roles_sid=realloc(r->roles_sid, (r->n_roles_sid+1)*sizeof(r->roles_sid[0])); |
|
|
794 | r->roles_sid[r->n_roles_sid++]=osm_protobufdb_string(ctx, role_buffer); |
|
|
795 | r->memids=realloc(r->memids, (r->n_memids+1)*sizeof(r->memids[0])); |
|
|
796 | r->memids[r->n_memids++]=atoll(ref_buffer); |
|
|
797 | r->types=realloc(r->types, (r->n_types+1)*sizeof(r->types[0])); |
|
|
798 | r->types[r->n_types++]=type; |
|
|
799 | } |
|
|
800 | return 1; |
|
|
801 | } |
|
|
802 | |
|
|
803 | |
|
|
804 | |
|
|
805 | int |
|
|
806 | osm_protobufdb_load(FILE *in, char *dir) |
|
|
807 | { |
|
|
808 | int size=BUFFER_SIZE; |
|
|
809 | char buffer[size]; |
|
|
810 | char *p; |
|
|
811 | sig_alrm(0); |
|
|
812 | test(); |
|
|
813 | while (fgets(buffer, size, in)) { |
|
|
814 | int closed=strstr(buffer,"/>")?1:0; |
|
|
815 | p=strchr(buffer,'<'); |
|
|
816 | if (! p) { |
|
|
817 | fprintf(stderr,"WARNING: wrong line %s\n", buffer); |
|
|
818 | continue; |
|
|
819 | } |
|
|
820 | if (!strncmp(p, "<?xml ",6)) { |
|
|
821 | } else if (!strncmp(p, "<osm ",5)) { |
|
|
822 | } else if (!strncmp(p, "<bound ",7)) { |
|
|
823 | } else if (!strncmp(p, "<node ",6)) { |
|
|
824 | if (!osm_protobufdb_parse_node(&context, p)) |
|
|
825 | fprintf(stderr,"WARNING: failed to parse %s\n", buffer); |
|
|
826 | if (closed) |
|
|
827 | osm_protobufdb_end_node(&context); |
|
|
828 | processed_nodes++; |
|
|
829 | } else if (!strncmp(p, "<tag ",5)) { |
|
|
830 | if (!osm_protobufdb_parse_tag(&context, p)) |
|
|
831 | fprintf(stderr,"WARNING: failed to parse %s\n", buffer); |
|
|
832 | } else if (!strncmp(p, "<way ",5)) { |
|
|
833 | if (!osm_protobufdb_parse_way(&context, p)) |
|
|
834 | fprintf(stderr,"WARNING: failed to parse %s\n", buffer); |
|
|
835 | if (closed) |
|
|
836 | osm_protobufdb_end_way(&context); |
|
|
837 | processed_ways++; |
|
|
838 | } else if (!strncmp(p, "<nd ",4)) { |
|
|
839 | if (!osm_protobufdb_parse_nd(&context, p)) |
|
|
840 | fprintf(stderr,"WARNING: failed to parse %s\n", buffer); |
|
|
841 | } else if (!strncmp(p, "<relation ",10)) { |
|
|
842 | if (!osm_protobufdb_parse_relation(&context, p)) |
|
|
843 | fprintf(stderr,"WARNING: failed to parse %s\n", buffer); |
|
|
844 | processed_relations++; |
|
|
845 | } else if (!strncmp(p, "<member ",8)) { |
|
|
846 | if (!osm_protobufdb_parse_member(&context, p)) |
|
|
847 | fprintf(stderr,"WARNING: failed to parse %s\n", buffer); |
|
|
848 | } else if (!strncmp(p, "</node>",7)) { |
|
|
849 | osm_protobufdb_end_node(&context); |
|
|
850 | } else if (!strncmp(p, "</way>",6)) { |
|
|
851 | osm_protobufdb_end_way(&context); |
|
|
852 | } else if (!strncmp(p, "</relation>",11)) { |
|
|
853 | osm_protobufdb_end_relation(&context); |
|
|
854 | } else if (!strncmp(p, "</osm>",6)) { |
|
|
855 | } else { |
|
|
856 | fprintf(stderr,"WARNING: unknown tag in %s\n", buffer); |
|
|
857 | } |
|
|
858 | } |
|
|
859 | finish(); |
|
|
860 | return 1; |
|
|
861 | } |
|
|