From afb9a0cad29cf9b6776c2a5cf885ae84bc4727ed Mon Sep 17 00:00:00 2001 From: feliperm17 Date: Thu, 16 Oct 2025 20:19:59 -0300 Subject: [PATCH 1/9] =?UTF-8?q?Implementa=20script=20para=20atualiza=C3=A7?= =?UTF-8?q?=C3=A3o=20de=20pol=C3=ADgonos=20de=20cidades=20com=20dados=20do?= =?UTF-8?q?=20IBGE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/scripts/update_polygons.js | 121 +++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 src/utils/scripts/update_polygons.js diff --git a/src/utils/scripts/update_polygons.js b/src/utils/scripts/update_polygons.js new file mode 100644 index 00000000..afc944ab --- /dev/null +++ b/src/utils/scripts/update_polygons.js @@ -0,0 +1,121 @@ +/* eslint-disable import/no-extraneous-dependencies */ +/* eslint-disable no-console */ +require('dotenv').config(); +const axios = require('axios'); +const { Client } = require('pg'); +const wkx = require('wkx'); + +const DB_USER = process.env.DB_USER || 'postgres'; +const DB_PASS = process.env.DB_PASS || 'postgres'; +const DB_HOST = process.env.DB_HOST || 'localhost'; +const DB_PORT = process.env.DB_PORT || '5432'; +const DB_NAME = process.env.DB_NAME || 'hcf'; + +const connectionString = `postgresql://${DB_USER}:${DB_PASS}@${DB_HOST}:${DB_PORT}/${DB_NAME}`; + +const API_IBGE_MUNICIPIOS = + 'https://servicodados.ibge.gov.br/api/v1/localidades/municipios'; +const API_IBGE_POLYGON = + 'https://servicodados.ibge.gov.br/api/v3/malhas/municipios/{codigo_ibge}?formato=application/vnd.geo+json&qualidade=minima'; + +function removerAcentos(str) { + return str.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); +} + +async function carregarListaMunicipios() { + const resp = await axios.get(API_IBGE_MUNICIPIOS, { timeout: 30000 }); + return resp.data.map(m => ({ + id: m.id, + nome: m.nome, + nome_normalizado: removerAcentos(m.nome.toLowerCase()), + })); +} + +function obterCodigoIbge(nome, municipios) { + const nomeNorm = removerAcentos(nome.toLowerCase()); + const m = municipios.find(x => x.nome_normalizado === nomeNorm); + return m ? parseInt(m.id, 10) : null; +} + +async function obterPoligonoIbge(codigoIbge) { + const url = API_IBGE_POLYGON.replace('{codigo_ibge}', codigoIbge); + const resp = await axios.get(url, { responseType: 'arraybuffer' }); + const geojson = JSON.parse(resp.data.toString('utf-8')); + + if (!geojson?.features?.length) throw new Error('GeoJSON vazio'); + + const { geometry } = geojson.features[0]; + if (!geometry) throw new Error('Sem geometria no GeoJSON'); + + const geom = wkx.Geometry.parseGeoJSON(geometry); + return geom.toWkb(); +} + +async function processarCidade(client, municipios, cidade) { + const { id, nome, pol_wkb: polWkb } = cidade; + + if (nome === 'Não Informado') return { inserido: 0, atualizado: 0, erro: 0 }; + + const codigoIbge = obterCodigoIbge(nome, municipios); + if (!codigoIbge) return { inserido: 0, atualizado: 0, erro: 1 }; + + let polBytes; + try { + polBytes = await obterPoligonoIbge(codigoIbge); + } catch { + return { inserido: 0, atualizado: 0, erro: 1 }; + } + + const sql = ` + WITH newgeom AS ( + SELECT ST_Multi(ST_SetSRID(ST_GeomFromWKB($1::bytea, 4674), 4674)) AS g + ) + UPDATE cidades + SET poligono = (SELECT g FROM newgeom), updated_at = NOW() + WHERE id = $2 + AND (poligono IS NULL OR NOT ST_Equals(poligono, (SELECT g FROM newgeom))); + `; + + try { + const result = await client.query(sql, [polBytes, id]); + if (result.rowCount > 0) { + return polWkb ? { inserido: 0, atualizado: 1, erro: 0 } : { inserido: 1, atualizado: 0, erro: 0 }; + } + return { inserido: 0, atualizado: 0, erro: 0 }; + } catch { + return { inserido: 0, atualizado: 0, erro: 1 }; + } +} + +async function main() { + const municipios = await carregarListaMunicipios(); + const client = new Client({ connectionString }); + await client.connect(); + + try { + const { rows: cidades } = await client.query( + 'SELECT id, nome, ST_AsBinary(poligono) AS pol_wkb FROM cidades;' + ); + + const resultados = await Promise.all( + cidades.map(c => processarCidade(client, municipios, c)) + ); + + const total = resultados.reduce( + (acc, r) => ({ + inseridos: acc.inseridos + r.inserido, + atualizados: acc.atualizados + r.atualizado, + erros: acc.erros + r.erro, + }), + { inseridos: 0, atualizados: 0, erros: 0 } + ); + + console.log('Resumo:', total); + } finally { + await client.end(); + } +} + +main().catch(err => { + console.error('Erro no processamento:', err); +}); From e9dcefe90559c2fede4ce9295dc65e32aae8a701 Mon Sep 17 00:00:00 2001 From: Moran Date: Tue, 4 Nov 2025 17:39:37 -0300 Subject: [PATCH 2/9] =?UTF-8?q?Adequa=C3=A7oes=20code=20review?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/scripts/update_polygons.js | 102 +++++++++++++++++++++------ 1 file changed, 80 insertions(+), 22 deletions(-) diff --git a/src/utils/scripts/update_polygons.js b/src/utils/scripts/update_polygons.js index afc944ab..d2b7912e 100644 --- a/src/utils/scripts/update_polygons.js +++ b/src/utils/scripts/update_polygons.js @@ -1,17 +1,50 @@ /* eslint-disable import/no-extraneous-dependencies */ /* eslint-disable no-console */ -require('dotenv').config(); -const axios = require('axios'); -const { Client } = require('pg'); -const wkx = require('wkx'); +import axios from 'axios'; +import dotenv from 'dotenv'; +import path from 'path'; +import pg from 'pg'; +import { fileURLToPath } from 'url'; +import wkx from 'wkx'; -const DB_USER = process.env.DB_USER || 'postgres'; -const DB_PASS = process.env.DB_PASS || 'postgres'; -const DB_HOST = process.env.DB_HOST || 'localhost'; -const DB_PORT = process.env.DB_PORT || '5432'; -const DB_NAME = process.env.DB_NAME || 'hcf'; +const currentFilename = fileURLToPath(import.meta.url); +const currentDirname = path.dirname(currentFilename); +const projectRoot = path.resolve(currentDirname, '..', '..', '..'); -const connectionString = `postgresql://${DB_USER}:${DB_PASS}@${DB_HOST}:${DB_PORT}/${DB_NAME}`; +dotenv.config({ path: path.join(projectRoot, '.env') }); + +const { Client } = pg; + +const { + PG_DATABASE, PG_USERNAME, PG_PASSWORD, PG_HOST, PG_PORT, +} = process.env; + +export const database = PG_DATABASE; +export const username = PG_USERNAME; +export const password = PG_PASSWORD; +export const host = PG_HOST; +export const port = PG_PORT; + +const connectionString = process.env.DATABASE_URL + || `postgresql://${PG_USERNAME}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${PG_DATABASE}`; + +function isValidConnectionString(cs) { + return typeof cs === 'string' && cs.length > 0 && !/undefined/.test(cs); +} + +if (!isValidConnectionString(connectionString)) { + console.error('Connection string inválida. Verifique as variáveis de ambiente PG_* ou DATABASE_URL.'); + console.error('Arquivo .env carregado em:', path.join(projectRoot, '.env')); + console.error('Valores atuais: PG_DATABASE=%s PG_USERNAME=%s PG_PASSWORD=%s PG_HOST=%s PG_PORT=%s', + PG_DATABASE, + PG_USERNAME, + PG_PASSWORD ? '***' : undefined, + PG_HOST, + PG_PORT); + process.exit(1); +} + +console.log('Using connectionString:', connectionString); const API_IBGE_MUNICIPIOS = 'https://servicodados.ibge.gov.br/api/v1/localidades/municipios'; @@ -39,13 +72,18 @@ function obterCodigoIbge(nome, municipios) { async function obterPoligonoIbge(codigoIbge) { const url = API_IBGE_POLYGON.replace('{codigo_ibge}', codigoIbge); - const resp = await axios.get(url, { responseType: 'arraybuffer' }); - const geojson = JSON.parse(resp.data.toString('utf-8')); + const resp = await axios.get(url, { responseType: 'arraybuffer', timeout: 30000 }); + const txt = resp.data.toString('utf-8'); + const geojson = JSON.parse(txt); - if (!geojson?.features?.length) throw new Error('GeoJSON vazio'); + if (!geojson?.features?.length) { + throw new Error(`GeoJSON vazio para codigo ${codigoIbge}`); + } const { geometry } = geojson.features[0]; - if (!geometry) throw new Error('Sem geometria no GeoJSON'); + if (!geometry) { + throw new Error(`Sem geometria no GeoJSON para codigo ${codigoIbge}`); + } const geom = wkx.Geometry.parseGeoJSON(geometry); return geom.toWkb(); @@ -54,10 +92,14 @@ async function obterPoligonoIbge(codigoIbge) { async function processarCidade(client, municipios, cidade) { const { id, nome, pol_wkb: polWkb } = cidade; - if (nome === 'Não Informado') return { inserido: 0, atualizado: 0, erro: 0 }; + if (nome === 'Não Informado') { + return { inserido: 0, atualizado: 0, erro: 0 }; + } const codigoIbge = obterCodigoIbge(nome, municipios); - if (!codigoIbge) return { inserido: 0, atualizado: 0, erro: 1 }; + if (!codigoIbge) { + return { inserido: 0, atualizado: 0, erro: 1 }; + } let polBytes; try { @@ -70,7 +112,7 @@ async function processarCidade(client, municipios, cidade) { WITH newgeom AS ( SELECT ST_Multi(ST_SetSRID(ST_GeomFromWKB($1::bytea, 4674), 4674)) AS g ) - UPDATE cidades + UPDATE public.cidades SET poligono = (SELECT g FROM newgeom), updated_at = NOW() WHERE id = $2 AND (poligono IS NULL OR NOT ST_Equals(poligono, (SELECT g FROM newgeom))); @@ -79,7 +121,9 @@ async function processarCidade(client, municipios, cidade) { try { const result = await client.query(sql, [polBytes, id]); if (result.rowCount > 0) { - return polWkb ? { inserido: 0, atualizado: 1, erro: 0 } : { inserido: 1, atualizado: 0, erro: 0 }; + return polWkb + ? { inserido: 0, atualizado: 1, erro: 0 } + : { inserido: 1, atualizado: 0, erro: 0 }; } return { inserido: 0, atualizado: 0, erro: 0 }; } catch { @@ -94,13 +138,26 @@ async function main() { try { const { rows: cidades } = await client.query( - 'SELECT id, nome, ST_AsBinary(poligono) AS pol_wkb FROM cidades;' + 'SELECT id, nome, ST_AsBinary(poligono) AS pol_wkb FROM public.cidades;' ); + const totalCidades = cidades.length; + let i = 0; const resultados = await Promise.all( - cidades.map(c => processarCidade(client, municipios, c)) + cidades.map(async c => { + i += 1; + const percentual = ((i / totalCidades) * 100).toFixed(1); + const restantes = totalCidades - i; + process.stdout.write(`\rProgresso: ${i}/${totalCidades} (${percentual}%) - Restam ${restantes} cidades`); + + const res = await processarCidade(client, municipios, c); + await new Promise((resolve) => { setTimeout(resolve, 100); }); + return res; + }) ); + process.stdout.write('\n'); + const total = resultados.reduce( (acc, r) => ({ inseridos: acc.inseridos + r.inserido, @@ -110,12 +167,13 @@ async function main() { { inseridos: 0, atualizados: 0, erros: 0 } ); - console.log('Resumo:', total); + console.log('\nResumo:', total); } finally { await client.end(); } } main().catch(err => { - console.error('Erro no processamento:', err); + console.error('\nErro no processamento:', err); + process.exit(1); }); From 6c360f22bd20b7f7009b2f4820e10dc60c4dfabb Mon Sep 17 00:00:00 2001 From: Moran <105233020+feliperm17@users.noreply.github.com> Date: Mon, 10 Nov 2025 22:53:43 -0300 Subject: [PATCH 3/9] Conversao Typescript --- src/utils/scripts/update_polygons.js | 179 ---------------------- src/utils/scripts/update_polygons.ts | 217 +++++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 179 deletions(-) delete mode 100644 src/utils/scripts/update_polygons.js create mode 100644 src/utils/scripts/update_polygons.ts diff --git a/src/utils/scripts/update_polygons.js b/src/utils/scripts/update_polygons.js deleted file mode 100644 index d2b7912e..00000000 --- a/src/utils/scripts/update_polygons.js +++ /dev/null @@ -1,179 +0,0 @@ -/* eslint-disable import/no-extraneous-dependencies */ -/* eslint-disable no-console */ -import axios from 'axios'; -import dotenv from 'dotenv'; -import path from 'path'; -import pg from 'pg'; -import { fileURLToPath } from 'url'; -import wkx from 'wkx'; - -const currentFilename = fileURLToPath(import.meta.url); -const currentDirname = path.dirname(currentFilename); -const projectRoot = path.resolve(currentDirname, '..', '..', '..'); - -dotenv.config({ path: path.join(projectRoot, '.env') }); - -const { Client } = pg; - -const { - PG_DATABASE, PG_USERNAME, PG_PASSWORD, PG_HOST, PG_PORT, -} = process.env; - -export const database = PG_DATABASE; -export const username = PG_USERNAME; -export const password = PG_PASSWORD; -export const host = PG_HOST; -export const port = PG_PORT; - -const connectionString = process.env.DATABASE_URL - || `postgresql://${PG_USERNAME}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${PG_DATABASE}`; - -function isValidConnectionString(cs) { - return typeof cs === 'string' && cs.length > 0 && !/undefined/.test(cs); -} - -if (!isValidConnectionString(connectionString)) { - console.error('Connection string inválida. Verifique as variáveis de ambiente PG_* ou DATABASE_URL.'); - console.error('Arquivo .env carregado em:', path.join(projectRoot, '.env')); - console.error('Valores atuais: PG_DATABASE=%s PG_USERNAME=%s PG_PASSWORD=%s PG_HOST=%s PG_PORT=%s', - PG_DATABASE, - PG_USERNAME, - PG_PASSWORD ? '***' : undefined, - PG_HOST, - PG_PORT); - process.exit(1); -} - -console.log('Using connectionString:', connectionString); - -const API_IBGE_MUNICIPIOS = - 'https://servicodados.ibge.gov.br/api/v1/localidades/municipios'; -const API_IBGE_POLYGON = - 'https://servicodados.ibge.gov.br/api/v3/malhas/municipios/{codigo_ibge}?formato=application/vnd.geo+json&qualidade=minima'; - -function removerAcentos(str) { - return str.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); -} - -async function carregarListaMunicipios() { - const resp = await axios.get(API_IBGE_MUNICIPIOS, { timeout: 30000 }); - return resp.data.map(m => ({ - id: m.id, - nome: m.nome, - nome_normalizado: removerAcentos(m.nome.toLowerCase()), - })); -} - -function obterCodigoIbge(nome, municipios) { - const nomeNorm = removerAcentos(nome.toLowerCase()); - const m = municipios.find(x => x.nome_normalizado === nomeNorm); - return m ? parseInt(m.id, 10) : null; -} - -async function obterPoligonoIbge(codigoIbge) { - const url = API_IBGE_POLYGON.replace('{codigo_ibge}', codigoIbge); - const resp = await axios.get(url, { responseType: 'arraybuffer', timeout: 30000 }); - const txt = resp.data.toString('utf-8'); - const geojson = JSON.parse(txt); - - if (!geojson?.features?.length) { - throw new Error(`GeoJSON vazio para codigo ${codigoIbge}`); - } - - const { geometry } = geojson.features[0]; - if (!geometry) { - throw new Error(`Sem geometria no GeoJSON para codigo ${codigoIbge}`); - } - - const geom = wkx.Geometry.parseGeoJSON(geometry); - return geom.toWkb(); -} - -async function processarCidade(client, municipios, cidade) { - const { id, nome, pol_wkb: polWkb } = cidade; - - if (nome === 'Não Informado') { - return { inserido: 0, atualizado: 0, erro: 0 }; - } - - const codigoIbge = obterCodigoIbge(nome, municipios); - if (!codigoIbge) { - return { inserido: 0, atualizado: 0, erro: 1 }; - } - - let polBytes; - try { - polBytes = await obterPoligonoIbge(codigoIbge); - } catch { - return { inserido: 0, atualizado: 0, erro: 1 }; - } - - const sql = ` - WITH newgeom AS ( - SELECT ST_Multi(ST_SetSRID(ST_GeomFromWKB($1::bytea, 4674), 4674)) AS g - ) - UPDATE public.cidades - SET poligono = (SELECT g FROM newgeom), updated_at = NOW() - WHERE id = $2 - AND (poligono IS NULL OR NOT ST_Equals(poligono, (SELECT g FROM newgeom))); - `; - - try { - const result = await client.query(sql, [polBytes, id]); - if (result.rowCount > 0) { - return polWkb - ? { inserido: 0, atualizado: 1, erro: 0 } - : { inserido: 1, atualizado: 0, erro: 0 }; - } - return { inserido: 0, atualizado: 0, erro: 0 }; - } catch { - return { inserido: 0, atualizado: 0, erro: 1 }; - } -} - -async function main() { - const municipios = await carregarListaMunicipios(); - const client = new Client({ connectionString }); - await client.connect(); - - try { - const { rows: cidades } = await client.query( - 'SELECT id, nome, ST_AsBinary(poligono) AS pol_wkb FROM public.cidades;' - ); - const totalCidades = cidades.length; - let i = 0; - - const resultados = await Promise.all( - cidades.map(async c => { - i += 1; - const percentual = ((i / totalCidades) * 100).toFixed(1); - const restantes = totalCidades - i; - process.stdout.write(`\rProgresso: ${i}/${totalCidades} (${percentual}%) - Restam ${restantes} cidades`); - - const res = await processarCidade(client, municipios, c); - await new Promise((resolve) => { setTimeout(resolve, 100); }); - return res; - }) - ); - - process.stdout.write('\n'); - - const total = resultados.reduce( - (acc, r) => ({ - inseridos: acc.inseridos + r.inserido, - atualizados: acc.atualizados + r.atualizado, - erros: acc.erros + r.erro, - }), - { inseridos: 0, atualizados: 0, erros: 0 } - ); - - console.log('\nResumo:', total); - } finally { - await client.end(); - } -} - -main().catch(err => { - console.error('\nErro no processamento:', err); - process.exit(1); -}); diff --git a/src/utils/scripts/update_polygons.ts b/src/utils/scripts/update_polygons.ts new file mode 100644 index 00000000..7fef6b32 --- /dev/null +++ b/src/utils/scripts/update_polygons.ts @@ -0,0 +1,217 @@ +/* eslint-disable no-console */ +import axios from 'axios'; +import dotenv from 'dotenv'; +import path from 'path'; +import pg from 'pg'; +import { fileURLToPath } from 'url'; +import wkx from 'wkx'; + +// === Tipos === +interface MunicipioIBGE { + id: number; + nome: string; + nome_normalizado: string; +} + +interface Cidade { + id: number; + nome: string; + pol_wkb: Buffer | null; +} + +interface ResultadoProcessamento { + inserido: number; + atualizado: number; + erro: number; +} + +// === Configuração de ambiente === +const currentFilename = fileURLToPath(import.meta.url); +const currentDirname = path.dirname(currentFilename); +const projectRoot = path.resolve(currentDirname, '..', '..', '..'); + +dotenv.config({ path: path.join(projectRoot, '.env') }); + +const { Client } = pg; +const { + PG_DATABASE, + PG_USERNAME, + PG_PASSWORD, + PG_HOST, + PG_PORT, + DATABASE_URL +} = process.env; + +export const database = PG_DATABASE; +export const username = PG_USERNAME; +export const password = PG_PASSWORD; +export const host = PG_HOST; +export const port = PG_PORT; + +const connectionString = + DATABASE_URL || + `postgresql://${PG_USERNAME}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${PG_DATABASE}`; + +function isValidConnectionString(cs: string | undefined): boolean { + return typeof cs === 'string' && cs.length > 0 && !/undefined/.test(cs); +} + +if (!isValidConnectionString(connectionString)) { + console.error('Connection string inválida. Verifique as variáveis de ambiente PG_* ou DATABASE_URL.'); + console.error('Arquivo .env carregado em:', path.join(projectRoot, '.env')); + console.error( + 'Valores atuais: PG_DATABASE=%s PG_USERNAME=%s PG_PASSWORD=%s PG_HOST=%s PG_PORT=%s', + PG_DATABASE, + PG_USERNAME, + PG_PASSWORD ? '***' : undefined, + PG_HOST, + PG_PORT + ); + process.exit(1); +} + +console.log('Using connectionString:', connectionString); + +// === URLs === +const API_IBGE_MUNICIPIOS = 'https://servicodados.ibge.gov.br/api/v1/localidades/municipios'; +const API_IBGE_POLYGON = + 'https://servicodados.ibge.gov.br/api/v3/malhas/municipios/{codigo_ibge}?formato=application/vnd.geo+json&qualidade=minima'; + +// === Funções auxiliares === +function removerAcentos(str: string): string { + return str.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); +} + +async function carregarListaMunicipios(): Promise { + const resp = await axios.get(API_IBGE_MUNICIPIOS, { timeout: 30000 }); + return resp.data.map((m: any) => ({ + id: m.id, + nome: m.nome, + nome_normalizado: removerAcentos(m.nome.toLowerCase()) + })); +} + +function obterCodigoIbge(nome: string, municipios: MunicipioIBGE[]): number | null { + const nomeNorm = removerAcentos(nome.toLowerCase()); + const m = municipios.find(x => x.nome_normalizado === nomeNorm); + return m ? parseInt(String(m.id), 10) : null; +} + +async function obterPoligonoIbge(codigoIbge: number): Promise { + const url = API_IBGE_POLYGON.replace('{codigo_ibge}', String(codigoIbge)); + const resp = await axios.get(url, { + responseType: 'arraybuffer', + timeout: 30000 + }); + + const txt = Buffer.from(resp.data).toString('utf-8'); + const geojson = JSON.parse(txt); + + if (!geojson?.features?.length) { + throw new Error(`GeoJSON vazio para codigo ${codigoIbge}`); + } + + const { geometry } = geojson.features[0]; + if (!geometry) { + throw new Error(`Sem geometria no GeoJSON para codigo ${codigoIbge}`); + } + + const geom = wkx.Geometry.parseGeoJSON(geometry); + return geom.toWkb(); +} + +async function processarCidade( + client: pg.Client, + municipios: MunicipioIBGE[], + cidade: Cidade +): Promise { + const { id, nome, pol_wkb: polWkb } = cidade; + + if (nome === 'Não Informado') { + return { inserido: 0, atualizado: 0, erro: 0 }; + } + + const codigoIbge = obterCodigoIbge(nome, municipios); + if (!codigoIbge) { + return { inserido: 0, atualizado: 0, erro: 1 }; + } + + let polBytes: Buffer; + try { + polBytes = await obterPoligonoIbge(codigoIbge); + } catch { + return { inserido: 0, atualizado: 0, erro: 1 }; + } + + const sql = ` + WITH newgeom AS ( + SELECT ST_Multi(ST_SetSRID(ST_GeomFromWKB($1::bytea, 4674), 4674)) AS g + ) + UPDATE public.cidades + SET poligono = (SELECT g FROM newgeom), updated_at = NOW() + WHERE id = $2 + AND (poligono IS NULL OR NOT ST_Equals(poligono, (SELECT g FROM newgeom))); + `; + + try { + const result = await client.query(sql, [polBytes, id]); + if (result.rowCount > 0) { + return polWkb + ? { inserido: 0, atualizado: 1, erro: 0 } + : { inserido: 1, atualizado: 0, erro: 0 }; + } + return { inserido: 0, atualizado: 0, erro: 0 }; + } catch { + return { inserido: 0, atualizado: 0, erro: 1 }; + } +} + +// === Execução principal === +async function main(): Promise { + const municipios = await carregarListaMunicipios(); + const client = new Client({ connectionString }); + await client.connect(); + + try { + const { rows: cidades } = await client.query( + 'SELECT id, nome, ST_AsBinary(poligono) AS pol_wkb FROM public.cidades;' + ); + const totalCidades = cidades.length; + let i = 0; + + const resultados = await Promise.all( + cidades.map(async c => { + i += 1; + const percentual = ((i / totalCidades) * 100).toFixed(1); + const restantes = totalCidades - i; + process.stdout.write( + `\rProgresso: ${i}/${totalCidades} (${percentual}%) - Restam ${restantes} cidades` + ); + + const res = await processarCidade(client, municipios, c); + await new Promise(resolve => setTimeout(resolve, 100)); + return res; + }) + ); + + process.stdout.write('\n'); + + const total = resultados.reduce( + (acc, r) => ({ + inseridos: acc.inseridos + r.inserido, + atualizados: acc.atualizados + r.atualizado, + erros: acc.erros + r.erro + }), + { inseridos: 0, atualizados: 0, erros: 0 } + ); + + console.log('\nResumo:', total); + } finally { + await client.end(); + } +} + +main().catch(err => { + console.error('\nErro no processamento:', err); + process.exit(1); +}); From 0b052f765913aaf4b56dd83e76127d417c54a4d8 Mon Sep 17 00:00:00 2001 From: Moran Date: Wed, 12 Nov 2025 18:44:43 -0300 Subject: [PATCH 4/9] Ajustes update_polygons --- package.json | 10 +- src/utils/scripts/update_polygons.ts | 179 +++++++++++---------------- 2 files changed, 79 insertions(+), 110 deletions(-) diff --git a/package.json b/package.json index 5363a3d9..8d8275f2 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "@babel/preset-typescript": "7.22.15", "@types/express": "5.0.1", "@types/jest": "30.0.0", + "@types/pg": "^8.15.6", "@types/react": "19.1.0", "@types/react-dom": "19.1.1", "@typescript-eslint/eslint-plugin": "6.7.0", @@ -64,12 +65,12 @@ "dotenv": "16.3.1", "ejs": "^2.6.1", "express": "5.1.0", + "express-rate-limit": "^7.4.1", "express-validator": "7.2.1", "fast-csv": "^5.0.5", "handlebars": "^4.7.8", "helmet": "^8.0.0", "jsonwebtoken": "9.0.2", - "express-rate-limit": "^7.4.1", "knex": "2.5.1", "moment": "^2.24.0", "moment-timezone": "^0.5.21", @@ -77,6 +78,8 @@ "multer": "1.4.5-lts.2", "mysql2": "3.14.0", "node-wkhtmltopdf": "^2.0.0", + "p-limit": "^7.2.0", + "pg": "^8.16.3", "puppeteer": "24.6.0", "q": "^1.5.1", "react": "19.1.0", @@ -88,7 +91,8 @@ "throttled-queue": "^1.0.5", "tsx": "4.19.3", "uuid": "11.1.0", - "wkhtmltopdf": "^0.4.0" + "wkhtmltopdf": "^0.4.0", + "wkx": "^0.5.0" }, "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e" -} \ No newline at end of file +} diff --git a/src/utils/scripts/update_polygons.ts b/src/utils/scripts/update_polygons.ts index 7fef6b32..712cc924 100644 --- a/src/utils/scripts/update_polygons.ts +++ b/src/utils/scripts/update_polygons.ts @@ -1,12 +1,12 @@ /* eslint-disable no-console */ -import axios from 'axios'; -import dotenv from 'dotenv'; -import path from 'path'; -import pg from 'pg'; -import { fileURLToPath } from 'url'; -import wkx from 'wkx'; - -// === Tipos === +import axios from 'axios' +import dotenv from 'dotenv' +import pLimit from 'p-limit' +import path from 'path' +import { Client } from 'pg' +import { fileURLToPath } from 'url' +import wkx from 'wkx' + interface MunicipioIBGE { id: number; nome: string; @@ -25,14 +25,12 @@ interface ResultadoProcessamento { erro: number; } -// === Configuração de ambiente === -const currentFilename = fileURLToPath(import.meta.url); -const currentDirname = path.dirname(currentFilename); -const projectRoot = path.resolve(currentDirname, '..', '..', '..'); +const currentFilename = fileURLToPath(import.meta.url) +const currentDirname = path.dirname(currentFilename) +const projectRoot = path.resolve(currentDirname, '..', '..', '..') -dotenv.config({ path: path.join(projectRoot, '.env') }); +dotenv.config({ path: path.join(projectRoot, '.env') }) -const { Client } = pg; const { PG_DATABASE, PG_USERNAME, @@ -40,107 +38,96 @@ const { PG_HOST, PG_PORT, DATABASE_URL -} = process.env; - -export const database = PG_DATABASE; -export const username = PG_USERNAME; -export const password = PG_PASSWORD; -export const host = PG_HOST; -export const port = PG_PORT; +} = process.env const connectionString = DATABASE_URL || - `postgresql://${PG_USERNAME}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${PG_DATABASE}`; + `postgresql://${PG_USERNAME}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${PG_DATABASE}` function isValidConnectionString(cs: string | undefined): boolean { - return typeof cs === 'string' && cs.length > 0 && !/undefined/.test(cs); + return typeof cs === 'string' && cs.length > 0 && !/undefined/.test(cs) } if (!isValidConnectionString(connectionString)) { - console.error('Connection string inválida. Verifique as variáveis de ambiente PG_* ou DATABASE_URL.'); - console.error('Arquivo .env carregado em:', path.join(projectRoot, '.env')); - console.error( - 'Valores atuais: PG_DATABASE=%s PG_USERNAME=%s PG_PASSWORD=%s PG_HOST=%s PG_PORT=%s', - PG_DATABASE, - PG_USERNAME, - PG_PASSWORD ? '***' : undefined, - PG_HOST, - PG_PORT - ); - process.exit(1); + process.exit(1) } -console.log('Using connectionString:', connectionString); - -// === URLs === -const API_IBGE_MUNICIPIOS = 'https://servicodados.ibge.gov.br/api/v1/localidades/municipios'; +const API_IBGE_MUNICIPIOS = 'https://servicodados.ibge.gov.br/api/v1/localidades/municipios' const API_IBGE_POLYGON = - 'https://servicodados.ibge.gov.br/api/v3/malhas/municipios/{codigo_ibge}?formato=application/vnd.geo+json&qualidade=minima'; + 'https://servicodados.ibge.gov.br/api/v3/malhas/municipios/{codigo_ibge}?formato=application/vnd.geo+json&qualidade=minima' + +const MAX_CONCURRENCY = 10 +const REQUEST_DELAY_MS = 150 -// === Funções auxiliares === function removerAcentos(str: string): string { - return str.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); + return str.normalize('NFD').replace(/[\u0300-\u036f]/g, '') +} + +function delay(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms) + }) } async function carregarListaMunicipios(): Promise { - const resp = await axios.get(API_IBGE_MUNICIPIOS, { timeout: 30000 }); + const resp = await axios.get(API_IBGE_MUNICIPIOS, { timeout: 30000 }) return resp.data.map((m: any) => ({ id: m.id, nome: m.nome, nome_normalizado: removerAcentos(m.nome.toLowerCase()) - })); + })) } function obterCodigoIbge(nome: string, municipios: MunicipioIBGE[]): number | null { - const nomeNorm = removerAcentos(nome.toLowerCase()); - const m = municipios.find(x => x.nome_normalizado === nomeNorm); - return m ? parseInt(String(m.id), 10) : null; + const nomeNorm = removerAcentos(nome.toLowerCase()) + const m = municipios.find(x => x.nome_normalizado === nomeNorm) + return m ? parseInt(String(m.id), 10) : null } async function obterPoligonoIbge(codigoIbge: number): Promise { - const url = API_IBGE_POLYGON.replace('{codigo_ibge}', String(codigoIbge)); + const url = API_IBGE_POLYGON.replace('{codigo_ibge}', String(codigoIbge)) const resp = await axios.get(url, { responseType: 'arraybuffer', timeout: 30000 - }); + }) - const txt = Buffer.from(resp.data).toString('utf-8'); - const geojson = JSON.parse(txt); + const txt = Buffer.from(resp.data).toString('utf-8') + const geojson = JSON.parse(txt) if (!geojson?.features?.length) { - throw new Error(`GeoJSON vazio para codigo ${codigoIbge}`); + throw new Error(`GeoJSON vazio para codigo ${codigoIbge}`) } - const { geometry } = geojson.features[0]; + const { geometry } = geojson.features[0] if (!geometry) { - throw new Error(`Sem geometria no GeoJSON para codigo ${codigoIbge}`); + throw new Error(`Sem geometria no GeoJSON para codigo ${codigoIbge}`) } - const geom = wkx.Geometry.parseGeoJSON(geometry); - return geom.toWkb(); + const geom = wkx.Geometry.parseGeoJSON(geometry) + return geom.toWkb() } async function processarCidade( - client: pg.Client, + client: Client, municipios: MunicipioIBGE[], cidade: Cidade ): Promise { - const { id, nome, pol_wkb: polWkb } = cidade; + const { id, nome, pol_wkb: polWkb } = cidade if (nome === 'Não Informado') { - return { inserido: 0, atualizado: 0, erro: 0 }; + return { inserido: 0, atualizado: 0, erro: 0 } } - const codigoIbge = obterCodigoIbge(nome, municipios); + const codigoIbge = obterCodigoIbge(nome, municipios) if (!codigoIbge) { - return { inserido: 0, atualizado: 0, erro: 1 }; + return { inserido: 0, atualizado: 0, erro: 1 } } - let polBytes: Buffer; + let polBytes: Buffer try { - polBytes = await obterPoligonoIbge(codigoIbge); + polBytes = await obterPoligonoIbge(codigoIbge) } catch { - return { inserido: 0, atualizado: 0, erro: 1 }; + return { inserido: 0, atualizado: 0, erro: 1 } } const sql = ` @@ -151,67 +138,45 @@ async function processarCidade( SET poligono = (SELECT g FROM newgeom), updated_at = NOW() WHERE id = $2 AND (poligono IS NULL OR NOT ST_Equals(poligono, (SELECT g FROM newgeom))); - `; + ` try { - const result = await client.query(sql, [polBytes, id]); - if (result.rowCount > 0) { + const result = await client.query(sql, [polBytes, id]) + if (result && result.rowCount && result.rowCount > 0) { return polWkb ? { inserido: 0, atualizado: 1, erro: 0 } - : { inserido: 1, atualizado: 0, erro: 0 }; + : { inserido: 1, atualizado: 0, erro: 0 } } - return { inserido: 0, atualizado: 0, erro: 0 }; + return { inserido: 0, atualizado: 0, erro: 0 } } catch { - return { inserido: 0, atualizado: 0, erro: 1 }; + return { inserido: 0, atualizado: 0, erro: 1 } } } -// === Execução principal === async function main(): Promise { - const municipios = await carregarListaMunicipios(); - const client = new Client({ connectionString }); - await client.connect(); + const municipios = await carregarListaMunicipios() + const client = new Client({ connectionString }) + await client.connect() + + const limit = pLimit(MAX_CONCURRENCY) try { const { rows: cidades } = await client.query( 'SELECT id, nome, ST_AsBinary(poligono) AS pol_wkb FROM public.cidades;' - ); - const totalCidades = cidades.length; - let i = 0; - - const resultados = await Promise.all( - cidades.map(async c => { - i += 1; - const percentual = ((i / totalCidades) * 100).toFixed(1); - const restantes = totalCidades - i; - process.stdout.write( - `\rProgresso: ${i}/${totalCidades} (${percentual}%) - Restam ${restantes} cidades` - ); - - const res = await processarCidade(client, municipios, c); - await new Promise(resolve => setTimeout(resolve, 100)); - return res; - }) - ); - - process.stdout.write('\n'); + ) - const total = resultados.reduce( - (acc, r) => ({ - inseridos: acc.inseridos + r.inserido, - atualizados: acc.atualizados + r.atualizado, - erros: acc.erros + r.erro - }), - { inseridos: 0, atualizados: 0, erros: 0 } - ); + const tarefas = cidades.map(c => + limit(async () => { + const res = await processarCidade(client, municipios, c) + await delay(REQUEST_DELAY_MS) + return res + }) + ) - console.log('\nResumo:', total); + await Promise.all(tarefas) } finally { - await client.end(); + await client.end() } } -main().catch(err => { - console.error('\nErro no processamento:', err); - process.exit(1); -}); +main().catch(() => process.exit(1)) From ee094630358a5d5d3bffd32542a37bf9ec3caf67 Mon Sep 17 00:00:00 2001 From: Moran Date: Wed, 12 Nov 2025 18:45:24 -0300 Subject: [PATCH 5/9] extra --- src/utils/scripts/update_polygons.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/scripts/update_polygons.ts b/src/utils/scripts/update_polygons.ts index 712cc924..93344cbb 100644 --- a/src/utils/scripts/update_polygons.ts +++ b/src/utils/scripts/update_polygons.ts @@ -64,7 +64,7 @@ function removerAcentos(str: string): string { } function delay(ms: number): Promise { - return new Promise((resolve) => { + return new Promise(resolve => { setTimeout(resolve, ms) }) } From 71fb5132ee119897398b60bd8f5ecbc7d92fb543 Mon Sep 17 00:00:00 2001 From: Moran Date: Fri, 28 Nov 2025 23:24:37 -0300 Subject: [PATCH 6/9] pg module --- yarn.lock | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/yarn.lock b/yarn.lock index 01b2e655..4672ef5d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1792,6 +1792,15 @@ dependencies: undici-types "~6.21.0" +"@types/pg@^8.15.6": + version "8.15.6" + resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.15.6.tgz#4df7590b9ac557cbe5479e0074ec1540cbddad9b" + integrity sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ== + dependencies: + "@types/node" "*" + pg-protocol "*" + pg-types "^2.2.0" + "@types/qs@*": version "6.9.18" resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.18.tgz#877292caa91f7c1b213032b34626505b746624c2" @@ -5946,6 +5955,13 @@ p-limit@^3.0.2, p-limit@^3.1.0: dependencies: yocto-queue "^0.1.0" +p-limit@^7.2.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-7.2.0.tgz#afcf6b5a86d093660140497dda0e640dd01a7b3b" + integrity sha512-ATHLtwoTNDloHRFFxFJdHnG6n2WUeFjaR8XQMFdKIv0xkXjrER8/iG9iu265jOM95zXHAfv9oTkqhrfbIzosrQ== + dependencies: + yocto-queue "^1.2.1" + p-locate@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz" @@ -6091,11 +6107,67 @@ performance-now@^2.1.0: resolved "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz" integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== +pg-cloudflare@^1.2.7: + version "1.2.7" + resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.2.7.tgz#a1f3d226bab2c45ae75ea54d65ec05ac6cfafbef" + integrity sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg== + pg-connection-string@2.6.1: version "2.6.1" resolved "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz" integrity sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg== +pg-connection-string@^2.9.1: + version "2.9.1" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.9.1.tgz#bb1fd0011e2eb76ac17360dc8fa183b2d3465238" + integrity sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w== + +pg-int8@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" + integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== + +pg-pool@^3.10.1: + version "3.10.1" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.10.1.tgz#481047c720be2d624792100cac1816f8850d31b2" + integrity sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg== + +pg-protocol@*, pg-protocol@^1.10.3: + version "1.10.3" + resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.10.3.tgz#ac9e4778ad3f84d0c5670583bab976ea0a34f69f" + integrity sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ== + +pg-types@2.2.0, pg-types@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" + integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== + dependencies: + pg-int8 "1.0.1" + postgres-array "~2.0.0" + postgres-bytea "~1.0.0" + postgres-date "~1.0.4" + postgres-interval "^1.1.0" + +pg@^8.16.3: + version "8.16.3" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.16.3.tgz#160741d0b44fdf64680e45374b06d632e86c99fd" + integrity sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw== + dependencies: + pg-connection-string "^2.9.1" + pg-pool "^3.10.1" + pg-protocol "^1.10.3" + pg-types "2.2.0" + pgpass "1.0.5" + optionalDependencies: + pg-cloudflare "^1.2.7" + +pgpass@1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" + integrity sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug== + dependencies: + split2 "^4.1.0" + picocolors@^1.0.0, picocolors@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz" @@ -6157,6 +6229,28 @@ possible-typed-array-names@^1.0.0: resolved "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz" integrity sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg== +postgres-array@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e" + integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA== + +postgres-bytea@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35" + integrity sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w== + +postgres-date@~1.0.4: + version "1.0.7" + resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8" + integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q== + +postgres-interval@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695" + integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ== + dependencies: + xtend "^4.0.0" + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz" @@ -6903,6 +6997,11 @@ spdx-license-ids@^3.0.0: resolved "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.21.tgz" integrity sha512-Bvg/8F5XephndSK3JffaRqdT+gyhfqIPwDHpX80tJrF8QQRYMo8sNMeaZ2Dp5+jhwKnUmIOyFFQfHRkjJm5nXg== +split2@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/split2/-/split2-4.2.0.tgz#c9c5920904d148bab0b9f67145f245a86aadbfa4" + integrity sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg== + sprintf-js@^1.1.3: version "1.1.3" resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz" @@ -7654,6 +7753,13 @@ wkx@^0.4.1: dependencies: "@types/node" "*" +wkx@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/wkx/-/wkx-0.5.0.tgz#c6c37019acf40e517cc6b94657a25a3d4aa33e8c" + integrity sha512-Xng/d4Ichh8uN4l0FToV/258EjMGU9MGcA0HV2d9B/ZpZB3lqQm7nkOdZdm5GhKtLLhAE7PiVQwN4eN+2YJJUg== + dependencies: + "@types/node" "*" + word-wrap@^1.2.5: version "1.2.5" resolved "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz" @@ -7747,6 +7853,11 @@ yocto-queue@^0.1.0: resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== +yocto-queue@^1.2.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.2.2.tgz#3e09c95d3f1aa89a58c114c99223edf639152c00" + integrity sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ== + z-schema@^5.0.1: version "5.0.6" resolved "https://registry.npmjs.org/z-schema/-/z-schema-5.0.6.tgz#46d6a687b15e4a4369e18d6cb1c7b8618fc256c5" From 3b7ddd4743ce51fe7b84d673c9c2211d791a3e3d Mon Sep 17 00:00:00 2001 From: Moran Date: Fri, 28 Nov 2025 23:40:35 -0300 Subject: [PATCH 7/9] modulo pg --- package.json | 2 ++ yarn.lock | 57 ++++++++++++++++++++++++++++++++++------------------ 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index e241385b..7ca7ef0c 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "audit:fix": "npm audit fix" }, "dependencies": { + "@types/pg": "^8.15.6", "axios": "1.13.2", "bcrypt": "6.0.0", "body-parser": "2.2.0", @@ -47,6 +48,7 @@ "multer": "2.0.2", "mysql2": "3.15.3", "nodemailer": "7.0.10", + "pg": "^8.16.3", "puppeteer": "24.28.0", "q": "1.5.1", "react": "19.2.0", diff --git a/yarn.lock b/yarn.lock index d81fd2b1..2a54bff1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5103,13 +5103,6 @@ p-limit@^3.0.2: dependencies: yocto-queue "^0.1.0" -p-limit@^7.2.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-7.2.0.tgz#afcf6b5a86d093660140497dda0e640dd01a7b3b" - integrity sha512-ATHLtwoTNDloHRFFxFJdHnG6n2WUeFjaR8XQMFdKIv0xkXjrER8/iG9iu265jOM95zXHAfv9oTkqhrfbIzosrQ== - dependencies: - yocto-queue "^1.2.1" - p-locate@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz" @@ -5264,6 +5257,11 @@ performance-now@^2.1.0: resolved "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz" integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== +pg-cloudflare@^1.2.7: + version "1.2.7" + resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.2.7.tgz#a1f3d226bab2c45ae75ea54d65ec05ac6cfafbef" + integrity sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg== + pg-connection-string@2.6.2: version "2.6.2" resolved "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz#713d82053de4e2bd166fab70cd4f26ad36aab475" @@ -5374,6 +5372,37 @@ possible-typed-array-names@^1.0.0: resolved "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz" integrity sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg== +postcss@^8.5.6: + version "8.5.6" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.5.6.tgz#2825006615a619b4f62a9e7426cc120b349a8f3c" + integrity sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg== + dependencies: + nanoid "^3.3.11" + picocolors "^1.1.1" + source-map-js "^1.2.1" + +postgres-array@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e" + integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA== + +postgres-bytea@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35" + integrity sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w== + +postgres-date@~1.0.4: + version "1.0.7" + resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8" + integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q== + +postgres-interval@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695" + integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ== + dependencies: + xtend "^4.0.0" + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz" @@ -6813,13 +6842,6 @@ wkx@^0.4.1: dependencies: "@types/node" "*" -wkx@^0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/wkx/-/wkx-0.5.0.tgz#c6c37019acf40e517cc6b94657a25a3d4aa33e8c" - integrity sha512-Xng/d4Ichh8uN4l0FToV/258EjMGU9MGcA0HV2d9B/ZpZB3lqQm7nkOdZdm5GhKtLLhAE7PiVQwN4eN+2YJJUg== - dependencies: - "@types/node" "*" - word-wrap@^1.2.5: version "1.2.5" resolved "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz" @@ -6844,7 +6866,7 @@ ws@^8.18.3: resolved "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz#b56b88abffde62791c639170400c93dcb0c95472" integrity sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg== -xtend@^4.0.2: +xtend@^4.0.0, xtend@^4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== @@ -6895,11 +6917,6 @@ yocto-queue@^0.1.0: resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== -yocto-queue@^1.2.1: - version "1.2.2" - resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.2.2.tgz#3e09c95d3f1aa89a58c114c99223edf639152c00" - integrity sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ== - z-schema@^5.0.1: version "5.0.6" resolved "https://registry.npmjs.org/z-schema/-/z-schema-5.0.6.tgz#46d6a687b15e4a4369e18d6cb1c7b8618fc256c5" From 36c8671ac8036a6d729b309faf51163f6da0eb16 Mon Sep 17 00:00:00 2001 From: Moran Date: Sat, 29 Nov 2025 00:12:51 -0300 Subject: [PATCH 8/9] =?UTF-8?q?Corre=C3=A7=C3=B5es=20lint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/scripts/update_polygons.ts | 71 ++++++++++++++++++---------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/src/utils/scripts/update_polygons.ts b/src/utils/scripts/update_polygons.ts index 93344cbb..e7b99dc4 100644 --- a/src/utils/scripts/update_polygons.ts +++ b/src/utils/scripts/update_polygons.ts @@ -8,21 +8,26 @@ import { fileURLToPath } from 'url' import wkx from 'wkx' interface MunicipioIBGE { - id: number; - nome: string; - nome_normalizado: string; + id: number + nome: string + nome_normalizado: string +} + +interface MunicipioIBGEResponse { + id: number + nome: string } interface Cidade { - id: number; - nome: string; - pol_wkb: Buffer | null; + id: number + nome: string + pol_wkb: Buffer | null } interface ResultadoProcessamento { - inserido: number; - atualizado: number; - erro: number; + inserido: number + atualizado: number + erro: number } const currentFilename = fileURLToPath(import.meta.url) @@ -40,9 +45,9 @@ const { DATABASE_URL } = process.env -const connectionString = - DATABASE_URL || - `postgresql://${PG_USERNAME}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${PG_DATABASE}` +const connectionString + = DATABASE_URL + || `postgresql://${PG_USERNAME}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${PG_DATABASE}` function isValidConnectionString(cs: string | undefined): boolean { return typeof cs === 'string' && cs.length > 0 && !/undefined/.test(cs) @@ -53,8 +58,8 @@ if (!isValidConnectionString(connectionString)) { } const API_IBGE_MUNICIPIOS = 'https://servicodados.ibge.gov.br/api/v1/localidades/municipios' -const API_IBGE_POLYGON = - 'https://servicodados.ibge.gov.br/api/v3/malhas/municipios/{codigo_ibge}?formato=application/vnd.geo+json&qualidade=minima' +const API_IBGE_POLYGON + = 'https://servicodados.ibge.gov.br/api/v3/malhas/municipios/{codigo_ibge}?formato=application/vnd.geo+json&qualidade=minima' const MAX_CONCURRENCY = 10 const REQUEST_DELAY_MS = 150 @@ -70,8 +75,8 @@ function delay(ms: number): Promise { } async function carregarListaMunicipios(): Promise { - const resp = await axios.get(API_IBGE_MUNICIPIOS, { timeout: 30000 }) - return resp.data.map((m: any) => ({ + const resp = await axios.get(API_IBGE_MUNICIPIOS, { timeout: 30000 }) + return resp.data.map((m: MunicipioIBGEResponse) => ({ id: m.id, nome: m.nome, nome_normalizado: removerAcentos(m.nome.toLowerCase()) @@ -92,7 +97,7 @@ async function obterPoligonoIbge(codigoIbge: number): Promise { }) const txt = Buffer.from(resp.data).toString('utf-8') - const geojson = JSON.parse(txt) + const geojson = JSON.parse(txt) as { features: Array<{ geometry: unknown }> } if (!geojson?.features?.length) { throw new Error(`GeoJSON vazio para codigo ${codigoIbge}`) @@ -112,22 +117,30 @@ async function processarCidade( municipios: MunicipioIBGE[], cidade: Cidade ): Promise { - const { id, nome, pol_wkb: polWkb } = cidade + const { + id, nome, pol_wkb: polWkb + } = cidade if (nome === 'Não Informado') { - return { inserido: 0, atualizado: 0, erro: 0 } + return { + inserido: 0, atualizado: 0, erro: 0 + } } const codigoIbge = obterCodigoIbge(nome, municipios) if (!codigoIbge) { - return { inserido: 0, atualizado: 0, erro: 1 } + return { + inserido: 0, atualizado: 0, erro: 1 + } } let polBytes: Buffer try { polBytes = await obterPoligonoIbge(codigoIbge) } catch { - return { inserido: 0, atualizado: 0, erro: 1 } + return { + inserido: 0, atualizado: 0, erro: 1 + } } const sql = ` @@ -144,12 +157,20 @@ async function processarCidade( const result = await client.query(sql, [polBytes, id]) if (result && result.rowCount && result.rowCount > 0) { return polWkb - ? { inserido: 0, atualizado: 1, erro: 0 } - : { inserido: 1, atualizado: 0, erro: 0 } + ? { + inserido: 0, atualizado: 1, erro: 0 + } + : { + inserido: 1, atualizado: 0, erro: 0 + } + } + return { + inserido: 0, atualizado: 0, erro: 0 } - return { inserido: 0, atualizado: 0, erro: 0 } } catch { - return { inserido: 0, atualizado: 0, erro: 1 } + return { + inserido: 0, atualizado: 0, erro: 1 + } } } From 6fc9694903cce8f8f090acf56f1578ccffa5d830 Mon Sep 17 00:00:00 2001 From: Moran Date: Sat, 20 Dec 2025 16:49:29 -0300 Subject: [PATCH 9/9] =?UTF-8?q?Altera=C3=A7=C3=A3o=20no=20script=20para=20?= =?UTF-8?q?buscar=20apenas=20poligonos=20para=20cidades=20brasileiras?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/scripts/update_polygons.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils/scripts/update_polygons.ts b/src/utils/scripts/update_polygons.ts index e7b99dc4..5c8d0e23 100644 --- a/src/utils/scripts/update_polygons.ts +++ b/src/utils/scripts/update_polygons.ts @@ -1,4 +1,3 @@ -/* eslint-disable no-console */ import axios from 'axios' import dotenv from 'dotenv' import pLimit from 'p-limit' @@ -183,7 +182,7 @@ async function main(): Promise { try { const { rows: cidades } = await client.query( - 'SELECT id, nome, ST_AsBinary(poligono) AS pol_wkb FROM public.cidades;' + 'SELECT c.id, c.nome, ST_AsBinary(c.poligono) AS pol_wkb FROM public.cidades c JOIN public.estados e ON c.estado_id = e.id WHERE e.pais_id = 76;' ) const tarefas = cidades.map(c =>