navit/maptool/osm_xml.c

/**
 * ZANavi, Zoff Android Navigation system.
 * Copyright (C) 2011-2012 Zoff <zoff@zoff.cc>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA  02110-1301, USA.
 */

/**
 * Navit, a modular navigation system.
 * Copyright (C) 2005-2011 Navit Team
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA  02110-1301, USA.
 */
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include <time.h>
#include <unistd.h>
#include "maptool.h"

int osm_xml_get_attribute(char *xml, char *attribute, char *buffer, int buffer_size)
{
        int len = strlen(attribute);
        char *pos, *i, s, attr[len + 2];
        strcpy(attr, attribute);
        strcpy(attr + len, "=");
        pos = strstr(xml, attr);
        if (!pos)
                return 0;
        pos += len + 1;
        s = *pos++;
        if (!s)
                return 0;
        i = strchr(pos, s);
        if (!i)
                return 0;
        if (i - pos > buffer_size)
        {
                fprintf(stderr, "Buffer overflow %ld vs %d\n", (long) (i - pos), buffer_size);
                return 0;
        }
        strncpy(buffer, pos, i - pos);
        buffer[i - pos] = '\0';
        return 1;
}

static struct entity
{
        char *entity;
        char c;
} entities[] = { { "&quot;", '"' }, { "&apos;", '\'' }, { "&amp;", '&' }, { "&lt;", '<' }, { "&gt;", '>' }, { "&#34;", '"' }, { "&#39;", '\'' }, { "&#38;", '&' }, { "&#60;", '<' }, { "&#62;", '>' }, { "&#123;", '{' }, { "&#125;", '}' }, };

void osm_xml_decode_entities(char *buffer)
{
        char *pos = buffer;
        int i, len, found;

        while ((pos = strchr(pos, '&')))
        {
                found = 0;
                for (i = 0; i < sizeof(entities) / sizeof(struct entity); i++)
                {
                        len = strlen(entities[i].entity);
                        if (!strncmp(pos, entities[i].entity, len))
                        {
                                *pos = entities[i].c;
                                memmove(pos + 1, pos + len, strlen(pos + len) + 1);
                                found = 1;
                                break;
                        }
                }
                pos++;
        }
}

static int parse_tag(char *p)
{
        char k_buffer[BUFFER_SIZE];
        char v_buffer[BUFFER_SIZE];
        if (!osm_xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE))
        {
                return 0;
        }
        if (!osm_xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE))
        {
                return 0;
        }
        osm_xml_decode_entities(v_buffer);
        osm_add_tag(k_buffer, v_buffer);
        return 1;
}

static int parse_node(char *p)
{
        char id_buffer[BUFFER_SIZE];
        char lat_buffer[BUFFER_SIZE];
        char lon_buffer[BUFFER_SIZE];
        if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
                return 0;
        if (!osm_xml_get_attribute(p, "lat", lat_buffer, BUFFER_SIZE))
                return 0;
        if (!osm_xml_get_attribute(p, "lon", lon_buffer, BUFFER_SIZE))
                return 0;
        osm_add_node(atoll(id_buffer), atof(lat_buffer), atof(lon_buffer));
        return 1;
}

static int parse_way(char *p)
{
        char id_buffer[BUFFER_SIZE];

        if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
        {
                return 0;
        }

        osm_add_way(atoll(id_buffer));
        return 1;
}

static int parse_relation(char *p)
{
        char id_buffer[BUFFER_SIZE];
        if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
                return 0;
        osm_add_relation(atoll(id_buffer));
        return 1;
}

static int parse_member(char *p)
{
        char type_buffer[BUFFER_SIZE];
        char ref_buffer[BUFFER_SIZE];
        char role_buffer[BUFFER_SIZE];
        int type;
        if (!osm_xml_get_attribute(p, "type", type_buffer, BUFFER_SIZE))
                return 0;
        if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
                return 0;
        if (!osm_xml_get_attribute(p, "role", role_buffer, BUFFER_SIZE))
                return 0;
        if (!strcmp(type_buffer, "node"))
                type = 1;
        else if (!strcmp(type_buffer, "way"))
                type = 2;
        else if (!strcmp(type_buffer, "relation"))
                type = 3;
        else
        {
                //fprintf(stderr,"Unknown type %s\n",type_buffer);
                type = 0;
        }
        osm_add_member(type, atoll(ref_buffer), role_buffer);

        return 1;
}

static int parse_nd(char *p)
{
        char ref_buffer[BUFFER_SIZE];
        if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
        {
                return 0;
        }
        osm_add_nd(atoll(ref_buffer));

        return 1;
}

int map_collect_data_osm(FILE *in, struct maptool_osm *osm)
{
        int size = 1024 * 5; // 5kb
        // char buffer[size];
        char *buffer_ptr;
        char *p;

        time_t start_tt, end_tt;
        double diff_tt;
        long long diff2_tt;
        long long pos_in;
        long long pos_in_local;
        int _c = 0;
        int _e = 5000000;
        int first_rel = 1;
        int first_way = 1;
        int line_length = 0;

        int percent_int_1 = 0;
        int percent_int_2 = 0;
        int percent_int_3 = 0;
        int phase = 0; // 0:nodes 1:ways 2:relations

        buffer_ptr = malloc(size);

        //sig_alrm(0);

        // reset
        pos_in = 0;
        pos_in_local = 0;
        diff2_tt = 0;

        while (fgets(buffer_ptr, size, in))
        {
                // we just read "size" bytes from "in"

                line_length = strlen(buffer_ptr);

                pos_in = pos_in + line_length;
                pos_in_local = pos_in_local + line_length;

                if (_c == 0)
                {
                        time(&start_tt);
                        pos_in_local = 0;
                }
                _c++;

                p = strchr(buffer_ptr, '<');
                if (!p)
                {
                        //fprintf(stderr,"WARNING: wrong line %s\n", buffer_ptr);
                        continue;
                }
                if (!strncmp(p, "<?xml ", 6))
                {
                }
                else if (!strncmp(p, "<osm ", 5))
                {
                }
                else if (!strncmp(p, "<bound ", 7))
                {
                }
                else if (!strncmp(p, "<node ", 6))
                {
                        if (!parse_node(p))
                        {
                                //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
                        }
                        processed_nodes++;
                }
                else if (!strncmp(p, "<tag ", 5))
                {
                        if (!parse_tag(p))
                        {
                                //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
                        }
                }
                else if (!strncmp(p, "<way ", 5))
                {
                        if (first_way == 1)
                        {
                                first_way = 0;

                                phase = 1;

                                flush_nodes(1, 0); // flush remaining nodes to "coords.tmp" from "osm_end_node"
                                free_buffer("dummy", &node_buffer[0]); // and free the memory

                                sql_counter = 0;
                                sql_counter2 = 0;
                                sql_counter3 = 0;
                                sql_counter4 = 0;
                                sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
                                fprintf(stderr, "nodes processed:%lld\n", processed_nodes);

                                if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
                                {
                                        // reopen
                                        sql_db_close();
                                        sql_db_open();
                                        sql_db_init(0);
                                }

                                fprintf(stderr, "SQL: (create index 003) start\n");
                                sql_create_index003();
                                fprintf(stderr, "SQL: (create index 003) ready\n");

                                if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
                                {
                                        // reopen for indexes to be used
                                        sql_db_close();
                                        sql_db_open();
                                        sql_db_init(0);
                                }

                                fprintf(stderr, "SQL: (first way) COMMIT\n");

                        }

                        if (!parse_way(p))
                        {
                                //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
                        }
                        processed_ways++;
                }
                else if (!strncmp(p, "<nd ", 4))
                {
                        if (!parse_nd(p))
                        {
                                //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
                        }
                }
                else if (!strncmp(p, "<relation ", 10))
                {
                        if (first_rel == 1)
                        {
                                first_rel = 0;

                                phase = 2;

                                sql_counter = 0;
                                sql_counter2 = 0;
                                sql_counter3 = 0;
                                sql_counter4 = 0;

                                sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
                                sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);

                                fprintf(stderr, "ways processed:%lld\n", processed_ways);

                                if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
                                {
                                        // reopen
                                        sql_db_close();
                                        sql_db_open();
                                        sql_db_init(0);
                                }

                                fprintf(stderr, "SQL: (create index 001) start\n");
                                sql_create_index001();
                                fprintf(stderr, "SQL: (create index 001) ready\n");

                                if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
                                {
                                        // reopen for indexes to be used
                                        sql_db_close();
                                        sql_db_open();
                                        sql_db_init(0);
                                }

                                fprintf(stderr, "SQL: (first relation) COMMIT\n");
                        }

                        if (!parse_relation(p))
                        {
                                if (verbose_mode)
                                        fprintf(stderr, "WARNING: failed to parse %s\n", buffer_ptr);
                        }
                        processed_relations++;
                }
                else if (!strncmp(p, "<member ", 8))
                {
                        if (!parse_member(p))
                        {
                                //fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
                        }
                }
                else if (!strncmp(p, "</node>", 7))
                {
                        osm_end_node(osm);
                }
                else if (!strncmp(p, "</way>", 6))
                {
                        osm_end_way(osm);
                }
                else if (!strncmp(p, "</relation>", 11))
                {
                        osm_end_relation(osm);
                }
                else if (!strncmp(p, "</osm>", 6))
                {
                }
                else
                {
                        // fprintf(stderr,"WARNING: unknown tag in %s\n", buffer_ptr);
                }

                if (_c > _e)
                {
                        _c = 0;

                        time(&end_tt);
                        diff_tt = difftime(end_tt, start_tt);
                        diff2_tt = diff2_tt + (long) diff_tt;

                        char outstring[200];
                        char outstring2[200];
                        convert_to_human_time(diff2_tt, outstring);
                        convert_to_human_bytes(pos_in, outstring2);
                        fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s elapsed (%s read)\n", outstring, outstring2);
                        convert_to_human_bytes((pos_in / diff2_tt), outstring2);
                        // fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s/s read\n", outstring2);

                        convert_to_human_bytes((pos_in_local / diff_tt), outstring2);
                        fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA (local loop): %s/s read\n", outstring2);

                        if (phase == 0)
                        {
                                if (processed_nodes_sum > 0)
                                {
                                        percent_int_1 = (int)((processed_nodes * 100) / processed_nodes_sum);
                                }

                                // fprintf_(stderr, "sum=%d nodes=%d\n", (int)processed_nodes_sum, (int)processed_nodes);
                        }
                        else if (phase == 1)
                        {
                                percent_int_1 = 100;
                                if (processed_ways_sum > 0)
                                {
                                        percent_int_2 = (int)((processed_ways * 100) / processed_ways_sum);
                                }
                        }
                        else if (phase == 2)
                        {
                                percent_int_2 = 100;
                                if (processed_relations_sum > 0)
                                {
                                        percent_int_3 = (int)((processed_relations * 100) / processed_relations_sum);
                                }
                        }

                        fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: [%d% | %d% | %d%] read\n", percent_int_1, percent_int_2, percent_int_3);
                }
        }

        // just in case, commit all we got left over
        //if (sql_counter > 0)
        //{
        sql_counter = 0;
        sql_counter2 = 0;
        sql_counter3 = 0;
        sql_counter4 = 0;
        sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
        sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
        sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
        sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
        sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
        sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
        sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
        sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
        sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
        fprintf(stderr, "SQL: (final) COMMIT\n");
        fprintf(stderr, "relations processed:%d\n", processed_relations);

        //}

        //sig_alrm(0);
        //sig_alrm_end();

        processed_relations_sum = processed_relations;
        processed_ways_sum = processed_ways;
        processed_nodes_sum = processed_nodes;
        

        free(buffer_ptr);

        return 1;
}

1	zoff99	8	/**
2	zoff99	37	* ZANavi, Zoff Android Navigation system.
3			* Copyright (C) 2011-2012 Zoff <zoff@zoff.cc>
4			*
5			* This program is free software; you can redistribute it and/or
6			* modify it under the terms of the GNU General Public License
7			* version 2 as published by the Free Software Foundation.
8			*
9			* This program is distributed in the hope that it will be useful,
10			* but WITHOUT ANY WARRANTY; without even the implied warranty of
11			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12			* GNU General Public License for more details.
13			*
14			* You should have received a copy of the GNU General Public License
15			* along with this program; if not, write to the
16			* Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17			* Boston, MA 02110-1301, USA.
18			*/
19
20			/**
21	zoff99	8	* Navit, a modular navigation system.
22			* Copyright (C) 2005-2011 Navit Team
23			*
24			* This program is free software; you can redistribute it and/or
25			* modify it under the terms of the GNU General Public License
26			* version 2 as published by the Free Software Foundation.
27			*
28			* This program is distributed in the hope that it will be useful,
29			* but WITHOUT ANY WARRANTY; without even the implied warranty of
30			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31			* GNU General Public License for more details.
32			*
33			* You should have received a copy of the GNU General Public License
34			* along with this program; if not, write to the
35			* Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
36			* Boston, MA 02110-1301, USA.
37			*/
38			#include <string.h>
39			#include <stdlib.h>
40			#include <math.h>
41	zoff99	37	#include <time.h>
42	zoff99	8	#include <unistd.h>
43			#include "maptool.h"
44
45	zoff99	37	int osm_xml_get_attribute(char xml, char attribute, char *buffer, int buffer_size)
46	zoff99	8	{
47	zoff99	37	int len = strlen(attribute);
48			char pos, i, s, attr[len + 2];
49	zoff99	8	strcpy(attr, attribute);
50	zoff99	37	strcpy(attr + len, "=");
51			pos = strstr(xml, attr);
52			if (!pos)
53	zoff99	8	return 0;
54	zoff99	37	pos += len + 1;
55			s = *pos++;
56			if (!s)
57	zoff99	8	return 0;
58	zoff99	37	i = strchr(pos, s);
59			if (!i)
60	zoff99	8	return 0;
61	zoff99	37	if (i - pos > buffer_size)
62			{
63			fprintf(stderr, "Buffer overflow %ld vs %d\n", (long) (i - pos), buffer_size);
64	zoff99	8	return 0;
65			}
66	zoff99	37	strncpy(buffer, pos, i - pos);
67			buffer[i - pos] = '\0';
68	zoff99	8	return 1;
69			}
70
71	zoff99	37	static struct entity
72			{
73	zoff99	8	char *entity;
74			char c;
75	zoff99	37	} entities[] = { { """, '"' }, { "'", '\'' }, { "&", '&' }, { "<", '<' }, { ">", '>' }, { """, '"' }, { "'", '\'' }, { "&", '&' }, { "<", '<' }, { ">", '>' }, { "{", '{' }, { "}", '}' }, };
76	zoff99	8
77	zoff99	37	void osm_xml_decode_entities(char *buffer)
78	zoff99	8	{
79	zoff99	37	char *pos = buffer;
80			int i, len, found;
81	zoff99	8
82	zoff99	37	while ((pos = strchr(pos, '&')))
83			{
84			found = 0;
85			for (i = 0; i < sizeof(entities) / sizeof(struct entity); i++)
86			{
87			len = strlen(entities[i].entity);
88			if (!strncmp(pos, entities[i].entity, len))
89			{
90			*pos = entities[i].c;
91			memmove(pos + 1, pos + len, strlen(pos + len) + 1);
92			found = 1;
93	zoff99	8	break;
94			}
95			}
96			pos++;
97			}
98			}
99
100	zoff99	37	static int parse_tag(char *p)
101	zoff99	8	{
102			char k_buffer[BUFFER_SIZE];
103			char v_buffer[BUFFER_SIZE];
104			if (!osm_xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE))
105	zoff99	37	{
106	zoff99	8	return 0;
107	zoff99	37	}
108	zoff99	8	if (!osm_xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE))
109	zoff99	37	{
110	zoff99	8	return 0;
111	zoff99	37	}
112	zoff99	8	osm_xml_decode_entities(v_buffer);
113			osm_add_tag(k_buffer, v_buffer);
114			return 1;
115			}
116
117	zoff99	37	static int parse_node(char *p)
118	zoff99	8	{
119			char id_buffer[BUFFER_SIZE];
120			char lat_buffer[BUFFER_SIZE];
121			char lon_buffer[BUFFER_SIZE];
122			if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
123			return 0;
124			if (!osm_xml_get_attribute(p, "lat", lat_buffer, BUFFER_SIZE))
125			return 0;
126			if (!osm_xml_get_attribute(p, "lon", lon_buffer, BUFFER_SIZE))
127			return 0;
128			osm_add_node(atoll(id_buffer), atof(lat_buffer), atof(lon_buffer));
129			return 1;
130			}
131
132	zoff99	37	static int parse_way(char *p)
133	zoff99	8	{
134			char id_buffer[BUFFER_SIZE];
135	zoff99	57
136	zoff99	8	if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
137	zoff99	57	{
138	zoff99	8	return 0;
139	zoff99	57	}
140
141	zoff99	8	osm_add_way(atoll(id_buffer));
142			return 1;
143			}
144
145	zoff99	37	static int parse_relation(char *p)
146	zoff99	8	{
147			char id_buffer[BUFFER_SIZE];
148			if (!osm_xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
149			return 0;
150			osm_add_relation(atoll(id_buffer));
151			return 1;
152			}
153
154	zoff99	37	static int parse_member(char *p)
155	zoff99	8	{
156			char type_buffer[BUFFER_SIZE];
157			char ref_buffer[BUFFER_SIZE];
158			char role_buffer[BUFFER_SIZE];
159			int type;
160			if (!osm_xml_get_attribute(p, "type", type_buffer, BUFFER_SIZE))
161			return 0;
162			if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
163			return 0;
164			if (!osm_xml_get_attribute(p, "role", role_buffer, BUFFER_SIZE))
165			return 0;
166	zoff99	37	if (!strcmp(type_buffer, "node"))
167			type = 1;
168			else if (!strcmp(type_buffer, "way"))
169			type = 2;
170			else if (!strcmp(type_buffer, "relation"))
171			type = 3;
172			else
173			{
174	zoff99	36	//fprintf(stderr,"Unknown type %s\n",type_buffer);
175	zoff99	37	type = 0;
176	zoff99	8	}
177			osm_add_member(type, atoll(ref_buffer), role_buffer);
178	zoff99	37
179	zoff99	8	return 1;
180			}
181
182	zoff99	37	static int parse_nd(char *p)
183	zoff99	8	{
184			char ref_buffer[BUFFER_SIZE];
185			if (!osm_xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
186	zoff99	37	{
187	zoff99	8	return 0;
188	zoff99	37	}
189	zoff99	8	osm_add_nd(atoll(ref_buffer));
190	zoff99	37
191	zoff99	8	return 1;
192			}
193
194	zoff99	37	int map_collect_data_osm(FILE in, struct maptool_osm osm)
195	zoff99	8	{
196	zoff99	37	int size = 1024 * 5; // 5kb
197			// char buffer[size];
198			char *buffer_ptr;
199	zoff99	8	char *p;
200	zoff99	37
201			time_t start_tt, end_tt;
202			double diff_tt;
203			long long diff2_tt;
204			long long pos_in;
205			long long pos_in_local;
206			int _c = 0;
207			int _e = 5000000;
208			int first_rel = 1;
209			int first_way = 1;
210			int line_length = 0;
211
212	zoff99	57	int percent_int_1 = 0;
213			int percent_int_2 = 0;
214			int percent_int_3 = 0;
215			int phase = 0; // 0:nodes 1:ways 2:relations
216
217	zoff99	37	buffer_ptr = malloc(size);
218
219			//sig_alrm(0);
220
221			// reset
222			pos_in = 0;
223			pos_in_local = 0;
224			diff2_tt = 0;
225
226			while (fgets(buffer_ptr, size, in))
227			{
228			// we just read "size" bytes from "in"
229
230			line_length = strlen(buffer_ptr);
231
232			pos_in = pos_in + line_length;
233			pos_in_local = pos_in_local + line_length;
234
235			if (_c == 0)
236			{
237			time(&start_tt);
238			pos_in_local = 0;
239			}
240			_c++;
241
242			p = strchr(buffer_ptr, '<');
243			if (!p)
244			{
245			//fprintf(stderr,"WARNING: wrong line %s\n", buffer_ptr);
246	zoff99	8	continue;
247			}
248	zoff99	37	if (!strncmp(p, "<?xml ", 6))
249			{
250			}
251			else if (!strncmp(p, "<osm ", 5))
252			{
253			}
254			else if (!strncmp(p, "<bound ", 7))
255			{
256			}
257			else if (!strncmp(p, "<node ", 6))
258			{
259	zoff99	8	if (!parse_node(p))
260	zoff99	36	{
261	zoff99	37	//fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
262	zoff99	36	}
263	zoff99	8	processed_nodes++;
264	zoff99	37	}
265			else if (!strncmp(p, "<tag ", 5))
266			{
267	zoff99	8	if (!parse_tag(p))
268	zoff99	36	{
269	zoff99	37	//fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
270	zoff99	36	}
271	zoff99	37	}
272			else if (!strncmp(p, "<way ", 5))
273			{
274			if (first_way == 1)
275			{
276			first_way = 0;
277
278	zoff99	57	phase = 1;
279
280	zoff99	37	flush_nodes(1, 0); // flush remaining nodes to "coords.tmp" from "osm_end_node"
281			free_buffer("dummy", &node_buffer[0]); // and free the memory
282
283			sql_counter = 0;
284			sql_counter2 = 0;
285			sql_counter3 = 0;
286			sql_counter4 = 0;
287			sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
288			sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
289			sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
290			sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
291			sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
292			sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
293			sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
294			sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
295			sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
296			fprintf(stderr, "nodes processed:%lld\n", processed_nodes);
297
298			if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
299			{
300			// reopen
301			sql_db_close();
302			sql_db_open();
303			sql_db_init(0);
304			}
305
306			fprintf(stderr, "SQL: (create index 003) start\n");
307			sql_create_index003();
308			fprintf(stderr, "SQL: (create index 003) ready\n");
309
310			if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
311			{
312			// reopen for indexes to be used
313			sql_db_close();
314			sql_db_open();
315			sql_db_init(0);
316			}
317
318			fprintf(stderr, "SQL: (first way) COMMIT\n");
319
320			}
321
322	zoff99	8	if (!parse_way(p))
323	zoff99	36	{
324	zoff99	37	//fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
325	zoff99	36	}
326	zoff99	8	processed_ways++;
327	zoff99	37	}
328			else if (!strncmp(p, "<nd ", 4))
329			{
330	zoff99	8	if (!parse_nd(p))
331	zoff99	36	{
332	zoff99	37	//fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
333	zoff99	36	}
334	zoff99	37	}
335			else if (!strncmp(p, "<relation ", 10))
336			{
337			if (first_rel == 1)
338			{
339			first_rel = 0;
340
341	zoff99	57	phase = 2;
342
343	zoff99	37	sql_counter = 0;
344			sql_counter2 = 0;
345			sql_counter3 = 0;
346			sql_counter4 = 0;
347
348			sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
349			sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
350			sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
351			sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
352			sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
353			sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
354			sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
355			sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
356			sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
357
358			fprintf(stderr, "ways processed:%lld\n", processed_ways);
359
360			if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
361			{
362			// reopen
363			sql_db_close();
364			sql_db_open();
365			sql_db_init(0);
366			}
367
368			fprintf(stderr, "SQL: (create index 001) start\n");
369			sql_create_index001();
370			fprintf(stderr, "SQL: (create index 001) ready\n");
371
372			if (! MAPTOOL_SQL_INPUT_TOO_SMALL)
373			{
374			// reopen for indexes to be used
375			sql_db_close();
376			sql_db_open();
377			sql_db_init(0);
378			}
379
380			fprintf(stderr, "SQL: (first relation) COMMIT\n");
381			}
382
383	zoff99	8	if (!parse_relation(p))
384	zoff99	36	{
385	zoff99	37	if (verbose_mode)
386			fprintf(stderr, "WARNING: failed to parse %s\n", buffer_ptr);
387	zoff99	36	}
388	zoff99	8	processed_relations++;
389	zoff99	37	}
390			else if (!strncmp(p, "<member ", 8))
391			{
392	zoff99	8	if (!parse_member(p))
393	zoff99	36	{
394	zoff99	37	//fprintf(stderr,"WARNING: failed to parse %s\n", buffer_ptr);
395	zoff99	36	}
396	zoff99	37	}
397			else if (!strncmp(p, "</node>", 7))
398			{
399	zoff99	36	osm_end_node(osm);
400	zoff99	37	}
401			else if (!strncmp(p, "</way>", 6))
402			{
403	zoff99	36	osm_end_way(osm);
404	zoff99	37	}
405			else if (!strncmp(p, "</relation>", 11))
406			{
407	zoff99	36	osm_end_relation(osm);
408	zoff99	8	}
409	zoff99	37	else if (!strncmp(p, "</osm>", 6))
410			{
411			}
412			else
413			{
414			// fprintf(stderr,"WARNING: unknown tag in %s\n", buffer_ptr);
415			}
416
417			if (_c > _e)
418			{
419			_c = 0;
420
421			time(&end_tt);
422			diff_tt = difftime(end_tt, start_tt);
423			diff2_tt = diff2_tt + (long) diff_tt;
424
425			char outstring[200];
426			char outstring2[200];
427			convert_to_human_time(diff2_tt, outstring);
428			convert_to_human_bytes(pos_in, outstring2);
429	zoff99	57	fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s elapsed (%s read)\n", outstring, outstring2);
430	zoff99	37	convert_to_human_bytes((pos_in / diff2_tt), outstring2);
431	zoff99	57	// fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: %s/s read\n", outstring2);
432	zoff99	37
433			convert_to_human_bytes((pos_in_local / diff_tt), outstring2);
434	zoff99	57	fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA (local loop): %s/s read\n", outstring2);
435
436			if (phase == 0)
437			{
438			if (processed_nodes_sum > 0)
439			{
440			percent_int_1 = (int)((processed_nodes * 100) / processed_nodes_sum);
441			}
442
443			// fprintf_(stderr, "sum=%d nodes=%d\n", (int)processed_nodes_sum, (int)processed_nodes);
444			}
445			else if (phase == 1)
446			{
447			percent_int_1 = 100;
448			if (processed_ways_sum > 0)
449			{
450			percent_int_2 = (int)((processed_ways * 100) / processed_ways_sum);
451			}
452			}
453			else if (phase == 2)
454			{
455			percent_int_2 = 100;
456			if (processed_relations_sum > 0)
457			{
458			percent_int_3 = (int)((processed_relations * 100) / processed_relations_sum);
459			}
460			}
461
462			fprintf_(stderr, "-RUNTIME-LOOP-COLLECT-DATA: [%d% \| %d% \| %d%] read\n", percent_int_1, percent_int_2, percent_int_3);
463	zoff99	37	}
464	zoff99	8	}
465	zoff99	37
466			// just in case, commit all we got left over
467			//if (sql_counter > 0)
468			//{
469			sql_counter = 0;
470			sql_counter2 = 0;
471			sql_counter3 = 0;
472			sql_counter4 = 0;
473			sqlite3_exec(sql_handle, "COMMIT", 0, 0, 0);
474			sqlite3_exec(sql_handle002a, "COMMIT", 0, 0, 0);
475			sqlite3_exec(sql_handle003a, "COMMIT", 0, 0, 0);
476			sqlite3_exec(sql_handle002b, "COMMIT", 0, 0, 0);
477			sqlite3_exec(sql_handle003b, "COMMIT", 0, 0, 0);
478			sqlite3_exec(sql_handle004, "COMMIT", 0, 0, 0);
479			sqlite3_exec(sql_handle005, "COMMIT", 0, 0, 0);
480			sqlite3_exec(sql_handle006, "COMMIT", 0, 0, 0);
481			sqlite3_exec(sql_handle007, "COMMIT", 0, 0, 0);
482			fprintf(stderr, "SQL: (final) COMMIT\n");
483			fprintf(stderr, "relations processed:%d\n", processed_relations);
484
485			//}
486
487			//sig_alrm(0);
488			//sig_alrm_end();
489
490	zoff99	57	processed_relations_sum = processed_relations;
491			processed_ways_sum = processed_ways;
492			processed_nodes_sum = processed_nodes;
493
494
495	zoff99	37	free(buffer_ptr);
496
497	zoff99	8	return 1;
498			}
499	zoff99	37