diff --git a/package.json b/package.json index e241385b..7ca7ef0c 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "audit:fix": "npm audit fix" }, "dependencies": { + "@types/pg": "^8.15.6", "axios": "1.13.2", "bcrypt": "6.0.0", "body-parser": "2.2.0", @@ -47,6 +48,7 @@ "multer": "2.0.2", "mysql2": "3.15.3", "nodemailer": "7.0.10", + "pg": "^8.16.3", "puppeteer": "24.28.0", "q": "1.5.1", "react": "19.2.0", diff --git a/src/utils/scripts/update_polygons.ts b/src/utils/scripts/update_polygons.ts new file mode 100644 index 00000000..5c8d0e23 --- /dev/null +++ b/src/utils/scripts/update_polygons.ts @@ -0,0 +1,202 @@ +import axios from 'axios' +import dotenv from 'dotenv' +import pLimit from 'p-limit' +import path from 'path' +import { Client } from 'pg' +import { fileURLToPath } from 'url' +import wkx from 'wkx' + +interface MunicipioIBGE { + id: number + nome: string + nome_normalizado: string +} + +interface MunicipioIBGEResponse { + id: number + nome: string +} + +interface Cidade { + id: number + nome: string + pol_wkb: Buffer | null +} + +interface ResultadoProcessamento { + inserido: number + atualizado: number + erro: number +} + +const currentFilename = fileURLToPath(import.meta.url) +const currentDirname = path.dirname(currentFilename) +const projectRoot = path.resolve(currentDirname, '..', '..', '..') + +dotenv.config({ path: path.join(projectRoot, '.env') }) + +const { + PG_DATABASE, + PG_USERNAME, + PG_PASSWORD, + PG_HOST, + PG_PORT, + DATABASE_URL +} = process.env + +const connectionString + = DATABASE_URL + || `postgresql://${PG_USERNAME}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${PG_DATABASE}` + +function isValidConnectionString(cs: string | undefined): boolean { + return typeof cs === 'string' && cs.length > 0 && !/undefined/.test(cs) +} + +if (!isValidConnectionString(connectionString)) { + process.exit(1) +} + +const API_IBGE_MUNICIPIOS = 'https://servicodados.ibge.gov.br/api/v1/localidades/municipios' +const API_IBGE_POLYGON + = 'https://servicodados.ibge.gov.br/api/v3/malhas/municipios/{codigo_ibge}?formato=application/vnd.geo+json&qualidade=minima' + +const MAX_CONCURRENCY = 10 +const REQUEST_DELAY_MS = 150 + +function removerAcentos(str: string): string { + return str.normalize('NFD').replace(/[\u0300-\u036f]/g, '') +} + +function delay(ms: number): Promise { + return new Promise(resolve => { + setTimeout(resolve, ms) + }) +} + +async function carregarListaMunicipios(): Promise { + const resp = await axios.get(API_IBGE_MUNICIPIOS, { timeout: 30000 }) + return resp.data.map((m: MunicipioIBGEResponse) => ({ + id: m.id, + nome: m.nome, + nome_normalizado: removerAcentos(m.nome.toLowerCase()) + })) +} + +function obterCodigoIbge(nome: string, municipios: MunicipioIBGE[]): number | null { + const nomeNorm = removerAcentos(nome.toLowerCase()) + const m = municipios.find(x => x.nome_normalizado === nomeNorm) + return m ? parseInt(String(m.id), 10) : null +} + +async function obterPoligonoIbge(codigoIbge: number): Promise { + const url = API_IBGE_POLYGON.replace('{codigo_ibge}', String(codigoIbge)) + const resp = await axios.get(url, { + responseType: 'arraybuffer', + timeout: 30000 + }) + + const txt = Buffer.from(resp.data).toString('utf-8') + const geojson = JSON.parse(txt) as { features: Array<{ geometry: unknown }> } + + if (!geojson?.features?.length) { + throw new Error(`GeoJSON vazio para codigo ${codigoIbge}`) + } + + const { geometry } = geojson.features[0] + if (!geometry) { + throw new Error(`Sem geometria no GeoJSON para codigo ${codigoIbge}`) + } + + const geom = wkx.Geometry.parseGeoJSON(geometry) + return geom.toWkb() +} + +async function processarCidade( + client: Client, + municipios: MunicipioIBGE[], + cidade: Cidade +): Promise { + const { + id, nome, pol_wkb: polWkb + } = cidade + + if (nome === 'Não Informado') { + return { + inserido: 0, atualizado: 0, erro: 0 + } + } + + const codigoIbge = obterCodigoIbge(nome, municipios) + if (!codigoIbge) { + return { + inserido: 0, atualizado: 0, erro: 1 + } + } + + let polBytes: Buffer + try { + polBytes = await obterPoligonoIbge(codigoIbge) + } catch { + return { + inserido: 0, atualizado: 0, erro: 1 + } + } + + const sql = ` + WITH newgeom AS ( + SELECT ST_Multi(ST_SetSRID(ST_GeomFromWKB($1::bytea, 4674), 4674)) AS g + ) + UPDATE public.cidades + SET poligono = (SELECT g FROM newgeom), updated_at = NOW() + WHERE id = $2 + AND (poligono IS NULL OR NOT ST_Equals(poligono, (SELECT g FROM newgeom))); + ` + + try { + const result = await client.query(sql, [polBytes, id]) + if (result && result.rowCount && result.rowCount > 0) { + return polWkb + ? { + inserido: 0, atualizado: 1, erro: 0 + } + : { + inserido: 1, atualizado: 0, erro: 0 + } + } + return { + inserido: 0, atualizado: 0, erro: 0 + } + } catch { + return { + inserido: 0, atualizado: 0, erro: 1 + } + } +} + +async function main(): Promise { + const municipios = await carregarListaMunicipios() + const client = new Client({ connectionString }) + await client.connect() + + const limit = pLimit(MAX_CONCURRENCY) + + try { + const { rows: cidades } = await client.query( + 'SELECT c.id, c.nome, ST_AsBinary(c.poligono) AS pol_wkb FROM public.cidades c JOIN public.estados e ON c.estado_id = e.id WHERE e.pais_id = 76;' + ) + + const tarefas = cidades.map(c => + limit(async () => { + const res = await processarCidade(client, municipios, c) + await delay(REQUEST_DELAY_MS) + return res + }) + ) + + await Promise.all(tarefas) + } finally { + await client.end() + } +} + +main().catch(() => process.exit(1)) diff --git a/yarn.lock b/yarn.lock index f11cdbae..2a54bff1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1810,6 +1810,15 @@ dependencies: undici-types "~6.21.0" +"@types/pg@^8.15.6": + version "8.15.6" + resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.15.6.tgz#4df7590b9ac557cbe5479e0074ec1540cbddad9b" + integrity sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ== + dependencies: + "@types/node" "*" + pg-protocol "*" + pg-types "^2.2.0" + "@types/qs@*": version "6.9.18" resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.18.tgz#877292caa91f7c1b213032b34626505b746624c2" @@ -5248,11 +5257,67 @@ performance-now@^2.1.0: resolved "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz" integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== +pg-cloudflare@^1.2.7: + version "1.2.7" + resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.2.7.tgz#a1f3d226bab2c45ae75ea54d65ec05ac6cfafbef" + integrity sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg== + pg-connection-string@2.6.2: version "2.6.2" resolved "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz#713d82053de4e2bd166fab70cd4f26ad36aab475" integrity sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA== +pg-connection-string@^2.9.1: + version "2.9.1" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.9.1.tgz#bb1fd0011e2eb76ac17360dc8fa183b2d3465238" + integrity sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w== + +pg-int8@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" + integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== + +pg-pool@^3.10.1: + version "3.10.1" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.10.1.tgz#481047c720be2d624792100cac1816f8850d31b2" + integrity sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg== + +pg-protocol@*, pg-protocol@^1.10.3: + version "1.10.3" + resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.10.3.tgz#ac9e4778ad3f84d0c5670583bab976ea0a34f69f" + integrity sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ== + +pg-types@2.2.0, pg-types@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" + integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== + dependencies: + pg-int8 "1.0.1" + postgres-array "~2.0.0" + postgres-bytea "~1.0.0" + postgres-date "~1.0.4" + postgres-interval "^1.1.0" + +pg@^8.16.3: + version "8.16.3" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.16.3.tgz#160741d0b44fdf64680e45374b06d632e86c99fd" + integrity sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw== + dependencies: + pg-connection-string "^2.9.1" + pg-pool "^3.10.1" + pg-protocol "^1.10.3" + pg-types "2.2.0" + pgpass "1.0.5" + optionalDependencies: + pg-cloudflare "^1.2.7" + +pgpass@1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" + integrity sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug== + dependencies: + split2 "^4.1.0" + picocolors@^1.0.0, picocolors@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz" @@ -5309,13 +5374,35 @@ possible-typed-array-names@^1.0.0: postcss@^8.5.6: version "8.5.6" - resolved "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz#2825006615a619b4f62a9e7426cc120b349a8f3c" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.5.6.tgz#2825006615a619b4f62a9e7426cc120b349a8f3c" integrity sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg== dependencies: nanoid "^3.3.11" picocolors "^1.1.1" source-map-js "^1.2.1" +postgres-array@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e" + integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA== + +postgres-bytea@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35" + integrity sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w== + +postgres-date@~1.0.4: + version "1.0.7" + resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8" + integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q== + +postgres-interval@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695" + integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ== + dependencies: + xtend "^4.0.0" + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz" @@ -6039,6 +6126,11 @@ spdx-license-ids@^3.0.0: resolved "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.21.tgz" integrity sha512-Bvg/8F5XephndSK3JffaRqdT+gyhfqIPwDHpX80tJrF8QQRYMo8sNMeaZ2Dp5+jhwKnUmIOyFFQfHRkjJm5nXg== +split2@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/split2/-/split2-4.2.0.tgz#c9c5920904d148bab0b9f67145f245a86aadbfa4" + integrity sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg== + sprintf-js@^1.1.3: version "1.1.3" resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz" @@ -6774,7 +6866,7 @@ ws@^8.18.3: resolved "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz#b56b88abffde62791c639170400c93dcb0c95472" integrity sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg== -xtend@^4.0.2: +xtend@^4.0.0, xtend@^4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==