From 72683a5fa255c756c017d30c7b9595dadf264662 Mon Sep 17 00:00:00 2001 From: Lucas Josino Date: Sat, 15 Jun 2024 21:21:56 -0300 Subject: [PATCH 1/6] refactor: delete js scripts --- package.json | 16 -- requests/output.json | 1 - requests/requests.json | 1 - scripts/constants.js | 58 ----- scripts/converter/json-to-sql.js | 214 ------------------ scripts/repositories/local-repository.js | 197 ---------------- scripts/repositories/remote-repository.js | 260 ---------------------- scripts/start.js | 159 ------------- scripts/utils.js | 8 - yarn.lock | 8 - 10 files changed, 922 deletions(-) delete mode 100644 package.json delete mode 100644 requests/output.json delete mode 100644 requests/requests.json delete mode 100644 scripts/constants.js delete mode 100644 scripts/converter/json-to-sql.js delete mode 100644 scripts/repositories/local-repository.js delete mode 100644 scripts/repositories/remote-repository.js delete mode 100644 scripts/start.js delete mode 100644 scripts/utils.js delete mode 100644 yarn.lock diff --git a/package.json b/package.json deleted file mode 100644 index b999bef..0000000 --- a/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "@hawapi/api-data", - "private": true, - "version": "1.0.0", - "type": "module", - "scripts": { - "start": "node -r dotenv/config scripts/start.js", - "dev": "node -r dotenv/config scripts/start.js dotenv_config_path=.env.dev --dev", - "convert": "node -r dotenv/config scripts/converter/json-to-sql.js", - "convert:dev": "node -r dotenv/config scripts/converter/json-to-sql.js dotenv_config_path=.env.dev", - "clear:dev": "rm -rf ./dev/" - }, - "dependencies": { - "dotenv": "^16.1.4" - } -} diff --git a/requests/output.json b/requests/output.json deleted file mode 100644 index 0967ef4..0000000 --- a/requests/output.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/requests/requests.json b/requests/requests.json deleted file mode 100644 index 0967ef4..0000000 --- a/requests/requests.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/scripts/constants.js b/scripts/constants.js deleted file mode 100644 index f89c628..0000000 --- a/scripts/constants.js +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Database folder variable defined on '.env' file - */ -export const DB_FOLDER = process.env.DB_FOLDER; - -/** - * API version variable defined on '.env' file - */ -export const VERSION = process.env.VERSION; - -/** - * API url variable defined on '.env' file - */ -export const URL = process.env.API_URL; - -/** - * API access token variable defined on '.env' file - */ -export const TOKEN = process.env.TOKEN; - -/** - * List of tables that require translations. - * - * All translations will be saved on 'data/translations/\_translations.json' - */ -export const TRANSLATION_TABLES = [ - "episodes", - "games", - "locations", - "seasons", - "overview", -]; - -/** - * List of fields that has translation on {@link TRANSLATION_TABLES}. - */ -export const TRANSLATION_FIELDS = [ - "language", - "title", - "name", - "description", - "trailer", - "genres", - "trailers", -]; - -/** - * List of tables from HawAPI. - */ -export const TABLES = [ - "actors", - "characters", - "episodes", - "games", - "locations", - "seasons", - "soundtracks", -]; diff --git a/scripts/converter/json-to-sql.js b/scripts/converter/json-to-sql.js deleted file mode 100644 index 2f24d40..0000000 --- a/scripts/converter/json-to-sql.js +++ /dev/null @@ -1,214 +0,0 @@ -import fs from "fs/promises"; - -import { DB_FOLDER, TRANSLATION_TABLES, VERSION } from "../constants.js"; -import { getSingular } from "../utils.js"; -import { log } from "console"; - -const SPECIAL_NAMES = ["character", "language"]; -const BASE_PATH = `./${DB_FOLDER}/${VERSION}/data`; -const OUTPUT_PATH = `${BASE_PATH}/sql`; - -/** - * Main script to convert json files into sqls - */ -async function main() { - const files = await fs.readdir(BASE_PATH, { - recursive: false, - }); - - // Create the folder path - await fs.mkdir(OUTPUT_PATH, { recursive: true }); - // Clear current file (if any) - await fs.writeFile(`${OUTPUT_PATH}/hawapi.sql`, ""); - - let queries = []; - for (let target of files) { - if (!(await fs.stat(`${BASE_PATH}/${target}`)).isDirectory()) { - // Remove file extension - target = target.substring(0, target.lastIndexOf(".")); - - let data = await getJsonFile(target, false); - - for (const item of data) { - let { fields, values, subQueries } = query(target, item); - - if (TRANSLATION_TABLES.includes(target)) { - let res = await translationQuery(target, item.uuid); - subQueries.push(...res); - } - - let sqlQuery = - `INSERT INTO ${target} (` + fields + ") VALUES (" + values + ");\n"; - - queries.push(sqlQuery, ...subQueries); - } - } - } - - await fs.writeFile(`${OUTPUT_PATH}/hawapi.sql`, queries); - - log( - `Converted '${BASE_PATH}' files into SQL (Output: '${OUTPUT_PATH}/hawapi.sql')` - ); -} - -/** - * Method that query all target translation - * @param {string} target The target (actors, episodes...) of the request - * @param {string} uuid The item identification - * @returns An SQL INSERT string with target translation - * @since 1.0.0 - */ -const translationQuery = async (target, uuid) => { - let data = await getJsonFile(target, true); - - let res = []; - for (const item of data) { - if (item[`${getSingular(target)}_uuid`] === uuid) { - let { fields, values } = query(target, item); - - let sqlQuery = - `INSERT INTO ${target}_translations (` + - fields + - ") VALUES (" + - values + - ");\n"; - - res.push(sqlQuery); - } - } - - return res; -}; - -/** - * Method that 'query' a field from main query - * @param {string} target The target (actors, episodes...) of the request - * @param {string} field The field name representing the 'sub query' - * @param {string[]} array All values from field of main query - * @returns An SQL INSERT string - * @since 1.0.0 - */ -const subQuery = (target, field, array) => { - let res = ""; - - for (const item of array) { - // Ignore any possible array or non dictionary/map - if (typeof item !== "object" || Array.isArray(item)) return ""; - - let { fields, values } = query(target, item); - - res += - `INSERT INTO ${target}_${field} (` + - fields + - ") VALUES (" + - values + - ");\n"; - } - - return res; -}; - -/** - * Method that query and create a SQL INSERT string - * @param {string} target The target (actors, episodes...) of the request - * @param {string[]} array All values from file/field - * @returns All fields, values and subQueries from file/field - * @since 1.0.0 - */ -const query = (target, array) => { - let subQueries = []; - let fields = ""; - let values = ""; - - for (const field in array) { - let value = array[field]; - - switch (typeof value) { - case "string": - values += `'${escapeSingleQuote(value)}', `; - break; - case "number": - case "boolean": - values += `${value}, `; - break; - case "object": - if (Array.isArray(value)) { - const res = subQuery(target, field, value); - if (res !== "") { - subQueries.push(res); - continue; - } - values += `'${objectToPostgreSQLArray(value)}', `; - } - break; - } - - if (SPECIAL_NAMES.includes(field)) { - fields += `"${field}", `; - continue; - } - - fields += `${field}, `; - } - - fields = fields.slice(0, -2); - values = values.slice(0, -2); - - return { - fields, - values, - subQueries, - }; -}; - -/** - * Method that converts all items inside the array into a PostgreSQL array - * @param {string[]} array All values from field - * @returns An string representing the PostgreSQL array - * @since 1.0.0 - */ -const objectToPostgreSQLArray = (array) => { - let res = "{"; - - for (let i = 0; i < array.length; i++) { - const value = array[i]; - - if (typeof value === "string") { - res += `"${value}", `; - continue; - } - - res += `${value}, `; - } - - return res.slice(0, -2) + "}"; -}; - -/** - * Method that read and parse the json file using default path - * @param {string} target The target (actors, episodes...) of the request - * @param {boolean} isTranslation Define target is from a normal or translation file - * @returns An JSON with all file items - * @since 1.0.0 - */ -const getJsonFile = async (target, isTranslation) => { - const res = await fs.readFile( - isTranslation - ? `${BASE_PATH}/translations/${target}_translations.json` - : `${BASE_PATH}/${target}.json` - ); - - return JSON.parse(res); -}; - -/** - * Method that escape/fix single quotes from string (according to SQL) - * @param {string} value The string to be fixed - * @returns An quote fixed string - */ -const escapeSingleQuote = (value) => { - return value.replace(/'/g, "''"); -}; - -main(); diff --git a/scripts/repositories/local-repository.js b/scripts/repositories/local-repository.js deleted file mode 100644 index f3e91ec..0000000 --- a/scripts/repositories/local-repository.js +++ /dev/null @@ -1,197 +0,0 @@ -import fs from "fs/promises"; -import { error, log, warn } from "console"; - -import { - TRANSLATION_FIELDS, - TRANSLATION_TABLES, - DB_FOLDER, - VERSION, -} from "../constants.js"; -import { getSingular } from "../utils.js"; - -/** - * Method that add a new item (to local repository) - * @param {string} target The target (actors, episodes...) of the request - * @param {string} body The body (model) of the request - * @since 1.0.0 - */ -export const insertLocal = async (target, body) => { - const path = `./${DB_FOLDER}/${VERSION}/data/${target}.json`; - try { - let data = await fs.readFile(path, "utf8"); - data = JSON.parse(data); - - if (TRANSLATION_TABLES.includes(target)) { - const translationFields = getTranslationFields(target, body); - - await insertLocalTranslation(target, translationFields); - - // Remove translation fields from main model - for (const field in translationFields) { - delete body[field]; - } - } - - data.push(body); - - await writeToFile(path, data); - } catch (err) { - error(err); - } -}; - -/** - * Method that add a new translation item (to local repository) - * @param {string} target The target (actors, episodes...) of the request - * @param {string} body The body (model) of the request - */ -export const insertLocalTranslation = async (target, body) => { - const path = `./${DB_FOLDER}/${VERSION}/data/translations/${target}_translations.json`; - try { - let data = await fs.readFile(path, "utf8"); - data = JSON.parse(data); - - data.push(body); - - await writeToFile(path, data); - } catch (err) { - error(err); - } -}; - -/** - * Method that update a item (from local repository) - * @param {string} target The target (actors, episodes...) of the request - * @param {string} body The body (model) of the request - * @since 1.0.0 - */ -export const updateLocal = async (target, patch) => { - const path = `./${DB_FOLDER}/${VERSION}/data/${target}.json`; - try { - let data = await fs.readFile(path, "utf8"); - data = JSON.parse(data); - - // Loop over all items inside file - for (const item in data) { - // Validate item - if (data[item].uuid === patch.uuid) { - // Only update fields declared inside 'patch' - for (const field in patch) { - if (TRANSLATION_FIELDS.includes(field)) { - warn( - `(UPDATE): Item '${patch.uuid}' contains translation fields that will be ignored.\n` + - `(UPDATE): To update translation fields use '${getSingular( - target - )}_uuid' and 'language' fields inside the request method` - ); - continue; - } - - data[item][field] = patch[field]; - } - } - } - - await writeToFile(path, data); - } catch (err) { - error(err); - } -}; - -export const updateLocalTranslation = async (target, patch) => { - const path = `./${DB_FOLDER}/${VERSION}/data/translations/${target}_translations.json`; - try { - let data = await fs.readFile(path, "utf8"); - data = JSON.parse(data); - - const uuidField = getSingular(target) + "_uuid"; - const uuid = patch[uuidField]; - for (const item in data) { - if ( - data[item][uuidField] === uuid && - data[item].language === patch.language - ) { - data[item][uuidField] = uuid; - - for (const field in patch) { - if (!TRANSLATION_FIELDS.includes(field) && field !== uuidField) { - warn( - `(UPDATE): Item '${uuid}' with language '${patch.language}' contains fields that will be ignored.\n` + - `(UPDATE): To update static fields use 'uuid' instead of '${getSingular( - target - )}_uuid' inside the request method` - ); - continue; - } - - data[item][field] = patch[field]; - } - } - } - - await writeToFile(path, data); - } catch (err) { - error(err); - } -}; - -/** - * Method that delete a item (from local repository) - * @param {string} uuid The identification of the item - * @param {string} target The target (actors, episodes...) of the request - * @since 1.0.0 - */ -export const deleteLocal = async (body, target) => { - const path = `./${DB_FOLDER}/${VERSION}/data/${target}.json`; - try { - let data = await fs.readFile(path, "utf8"); - data = JSON.parse(data); - - data = data.filter((e) => e.uuid !== body.uuid); - - if (TRANSLATION_TABLES.includes(target)) { - await deleteLocalTranslation(target, body); - } - - await writeToFile(path, data); - } catch (err) { - error(err); - } -}; - -export const deleteLocalTranslation = async (target, body) => { - const path = `./${DB_FOLDER}/${VERSION}/data/translations/${target}_translations.json`; - try { - let data = await fs.readFile(path, "utf8"); - data = JSON.parse(data); - - const uuid = body[getSingular(target) + "_uuid"]; - data = data.filter((e) => e.uuid !== uuid && e.language !== body.language); - - await writeToFile(path, data); - } catch (err) { - error(err); - } -}; - -/** - * Method that write the data to specific file - * @param {string} path The file path - * @param {string} data The file data - * @since 1.0.0 - */ -const writeToFile = async (path, data) => { - await fs.writeFile(path, JSON.stringify(data, null, 2)); -}; - -const getTranslationFields = (target, body) => { - let fields = {}; - for (const field in body) { - if (TRANSLATION_FIELDS.includes(field)) { - fields[field] = body[field]; - } - } - log(target); - fields[`${getSingular(target)}_uuid`] = body["uuid"]; - return fields; -}; diff --git a/scripts/repositories/remote-repository.js b/scripts/repositories/remote-repository.js deleted file mode 100644 index e41648d..0000000 --- a/scripts/repositories/remote-repository.js +++ /dev/null @@ -1,260 +0,0 @@ -import { error } from "console"; - -import { VERSION, URL, TOKEN, TRANSLATION_TABLES } from "../constants.js"; -import { - deleteLocal, - deleteLocalTranslation, - insertLocal, - insertLocalTranslation, - updateLocal, - updateLocalTranslation, -} from "./local-repository.js"; -import { getSingular } from "../utils.js"; - -// Default options -const options = { - mode: "cors", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${TOKEN}`, - }, -}; - -/** - * Method that add a new item - * @param {string} target The target (actors, episodes...) of the request - * @param {string} body The body (model) of the request - * @returns An 'status_code' and 'message' - * @since 1.0.0 - * @see {@link insertLocal} - */ -export const insert = async (target, body) => { - options["method"] = "POST"; - options["body"] = JSON.stringify(body); - - const result = await fetch(URL + `/${VERSION}/${target}`, options); - - if (result.ok) { - const resBody = await result.json(); - await insertLocal(target, resBody); - } - - return { - status_code: result.status, - message: await getMessageOr( - result, - `Something wrong happen. Item: '${target}` - ), - }; -}; - -/** - * Method that add a new translation item - * @param {string} target The target (actors, episodes...) of the request - * @param {string} body The body (model) of the request - * @returns An 'status_code' and 'message' - * @since 1.0.0 - * @see {@link insertLocalTranslation} - */ -export const insertTranslation = async (target, body) => { - const message = validateTranslation(target, body); - if (message) return message; - - options["method"] = "POST"; - options["body"] = JSON.stringify(body); - - const uuid = body[`${getSingular(target)}_uuid`]; - const result = await fetch( - URL + `/${VERSION}/${target}/${uuid}/translations`, - options - ); - - if (result.ok) { - const resBody = await result.json(); - resBody[`${getSingular(target)}_uuid`] = uuid; - await insertLocalTranslation(target, resBody); - } - - return { - status_code: result.status, - message: await getMessageOr( - result, - `Something wrong happen. Item: '${uuid}` - ), - }; -}; - -/** - * Method that update a item - * @param {string} target The target (actors, episodes...) of the request - * @param {string} body The body (model) of the request - * @returns An 'status_code' and 'message' - * @since 1.0.0 - * @see {@link updateLocal} - */ -export const updateBy = async (target, patch) => { - options["method"] = "PATCH"; - options["body"] = JSON.stringify(patch); - - const result = await fetch( - URL + `/${VERSION}/${target}/${patch.uuid}`, - options - ); - - if (result.ok) { - await updateLocal(target, patch); - } - - return { - status_code: result.status, - message: await getMessageOr( - result, - `Something wrong happen. Item: '${patch.uuid}` - ), - }; -}; - -/** - * Method that update a translation item - * @param {string} target The target (actors, episodes...) of the request - * @param {string} body The body (model) of the request - * @returns An 'status_code' and 'message' - * @since 1.0.0 - * @see {@link updateLocalTranslation} - */ -export const updateTranslation = async (target, patch) => { - const message = validateTranslation(target, patch); - if (message) return message; - - options["method"] = "PATCH"; - options["body"] = JSON.stringify(patch); - - const uuid = patch[`${getSingular(target)}_uuid`]; - const result = await fetch( - URL + `/${VERSION}/${target}/${uuid}/translations/${patch.language}`, - options - ); - - if (result.ok) { - await updateLocalTranslation(target, patch); - } - - return { - status_code: result.status, - message: await getMessageOr( - result, - `Something wrong happen. Item: '${uuid}` - ), - }; -}; - -/** - * Method that delete a item - * @param {string} target The target (actors, episodes...) of the request - * @param {string} body The body (model) of the request - * @returns An 'status_code' and 'message' - * @since 1.0.0 - * @see {@link deleteLocal} - */ -export const deleteBy = async (target, body) => { - options["method"] = "DELETE"; - - const result = await fetch( - URL + `/${VERSION}/${target}/${body.uuid}`, - options - ); - - let message = `Deleted item: '${body.uuid}'`; - if (result.ok) { - await deleteLocal(body, target); - } else { - message = await getMessageOr( - result, - `Something wrong happen. Item: '${uuid}` - ); - } - - return { - status_code: result.status, - message, - }; -}; - -/** - * Method that delete a translation item - * @param {string} target The target (actors, episodes...) of the request - * @param {string} body The body (model) of the request - * @returns An 'status_code' and 'message' - * @since 1.0.0 - * @see {@link deleteLocalTranslation} - */ -export const deleteTranslation = async (target, body) => { - let message = validateTranslation(target, body); - if (message) return message; - - options["method"] = "DELETE"; - - const uuid = body[`${getSingular(target)}_uuid`]; - const result = await fetch( - URL + `/${VERSION}/${target}/${uuid}/translations/${body.language}`, - options - ); - - message = `Deleted item: '${uuid}'`; - if (result.ok) { - await deleteLocalTranslation(target, body); - } else { - message = await getMessageOr( - result, - `Something wrong happen. Item: '${uuid} and Language '${body.language}'` - ); - } - - return { - status_code: result.status, - message, - }; -}; - -/** - * Method to get response body - * @param {Response} response - * @returns Response body or the fallback message - * @since 1.0.0 - */ -const getMessageOr = async (response, fallback) => { - let message = ""; - try { - message = await response.json(); - } catch (err) { - // Handle 2XX without body - if (response.ok) return null; - - error(err); - message = fallback; - } - return message; -}; - -/** - * Method to validate if request is for static or translation table - * @param {string} target The target (actors, episodes...) of the request - * @param {string} patch The body (model) of the request - * @returns An error message if NOT VALID or undefined if VALID - * @since 1.0.0 - */ -const validateTranslation = (target, patch) => { - if (!TRANSLATION_TABLES.includes(target)) { - return { - status_code: 0, - message: `Table '${target}' doesn't have a translation table`, - }; - } - - if (!("language" in patch)) { - return { - status_code: 0, - message: "Field 'language' not defined!", - }; - } -}; diff --git a/scripts/start.js b/scripts/start.js deleted file mode 100644 index ca1dc4d..0000000 --- a/scripts/start.js +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/node - -import { error, log } from "console"; -import fs from "fs"; - -import { - deleteBy, - deleteTranslation, - insert, - insertTranslation, - updateBy, - updateTranslation, -} from "./repositories/remote-repository.js"; -import { - DB_FOLDER, - TABLES, - TRANSLATION_TABLES, - URL, - VERSION, -} from "./constants.js"; -import { getSingular } from "./utils.js"; - -let output = {}; -let outputFile = "./requests/output.json"; -let requestsFile = "./requests/requests.json"; - -/** - * Main script - */ -async function main() { - // Test API connection - await ping(); - - // Script setup - await setup(); - - if (process.argv.includes("--dev")) { - outputFile = `./${DB_FOLDER}/output.json`; - requestsFile = `./${DB_FOLDER}/requests.json`; - } - - // Loop over all methods - const requests = await getRequests(); - for (const method in requests) { - // Loop over all targets - for (const target in requests[method]) { - // Create new target - output[method] = { ...output[method], [target]: [] }; - - // Loop over all items - for (const item in requests[method][target]) { - const value = requests[method][target][item]; - - let res; - let isTranslation = `${getSingular(target)}_uuid` in value; - switch (true) { - case isTranslation && method === "POST": - res = await insertTranslation(target, value); - break; - case method === "POST": - res = await insert(target, value); - break; - case isTranslation && method === "UPDATE": - res = await updateTranslation(target, value); - break; - case method === "UPDATE": - res = await updateBy(target, value); - break; - case isTranslation && method === "DELETE": - res = await deleteTranslation(target, value); - break; - case method === "DELETE": - res = await deleteBy(target, value); - break; - default: - console.warn(`Method '${method}' is not valid. Skipping`); - res = { - status_code: 0, - message: `Method '${method}' is not valid`, - }; - } - - output[method][target].push(res); - } - } - } - - fs.writeFile(outputFile, JSON.stringify(output, null, 2), (error) => { - if (error) throw error; - }); - - fs.writeFile(requestsFile, "{}", (error) => { - if (error) throw error; - }); -} - -/** - * Test API connection - */ -async function ping() { - await fetch(URL + "/ping") - .then(async (res) => { - log(`[${res.status}] ${await res.text()}`); - }) - .catch((err) => { - error(err); - process.exit(1); - }); -} - -/** - * Method to read requests file - * @returns All content inside the requests file - */ -const getRequests = async () => { - const res = await fs.promises.readFile(requestsFile, { encoding: "utf8" }); - return JSON.parse(res); -}; - -/** - * Script setup - * - * * Running for the **first time** will create all 'database' files. - */ -const setup = async () => { - if (!fs.existsSync(`./${DB_FOLDER}/`)) { - try { - await fs.promises.mkdir(`./${DB_FOLDER}/${VERSION}/data/translations`, { - recursive: true, - }); - - const options = { - recursive: true, - flag: "w+", - }; - - for (const table in TABLES) { - await fs.promises.writeFile( - `./${DB_FOLDER}/${VERSION}/data/${TABLES[table]}.json`, - "[]", - options - ); - } - - for (const table in TRANSLATION_TABLES) { - await fs.promises.writeFile( - `./${DB_FOLDER}/${VERSION}/data/translations/${TRANSLATION_TABLES[table]}_translations.json`, - "[]", - options - ); - } - } catch (err) { - error(err); - process.exit(1); - } - } -}; - -main(); diff --git a/scripts/utils.js b/scripts/utils.js deleted file mode 100644 index 483dc6b..0000000 --- a/scripts/utils.js +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Method to get singular name of table - * @param {string} value The table name - * @returns Singular version of table name - */ -export const getSingular = (value) => { - return value.slice(0, -1); -}; diff --git a/yarn.lock b/yarn.lock deleted file mode 100644 index dbdd218..0000000 --- a/yarn.lock +++ /dev/null @@ -1,8 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -dotenv@^16.1.4: - version "16.1.4" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.1.4.tgz#67ac1a10cd9c25f5ba604e4e08bc77c0ebe0ca8c" - integrity sha512-m55RtE8AsPeJBpOIFKihEmqUcoVncQIwo7x9U8ZwLEZw9ZpXboz2c+rvog+jUaJvVrZ5kBOeYQBX5+8Aa/OZQw== From e4168e944f8b5b279725d509973268d84fba8436 Mon Sep 17 00:00:00 2001 From: Lucas Josino Date: Sat, 15 Jun 2024 21:25:27 -0300 Subject: [PATCH 2/6] wip: new json to sql converter --- Makefile | 32 +++++++++++ cmd/main.go | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 3 + 3 files changed, 195 insertions(+) create mode 100644 Makefile create mode 100644 cmd/main.go create mode 100644 go.mod diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..beb5fed --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +# Makefile +.DEFAULT_GOAL := help + +# Colors +RED := $(shell tput -Txterm setaf 1) +GREEN := $(shell tput -Txterm setaf 2) +BLUE := $(shell tput -Txterm setaf 4) +CYAN := $(shell tput -Txterm setaf 6) +RESET := $(shell tput -Txterm sgr0) + +## Commands + +run: ## Run the application + @go run cmd/main.go + +clean: ## Clean the application output files + @rm -rf database/v1/data/sql + +## Help + +# https://gist.github.com/thomaspoignant/5b72d579bd5f311904d973652180c705 +help: ## Show this help + @echo + @echo 'Usage:' + @echo ' ${CYAN}make${RESET} ${GREEN}${RESET}' + @echo + @echo 'Targets:' + @awk 'BEGIN {FS = ":.*?## "} { \ + if (/[a-zA-Z_\-]+:.*?##.*$$/) {printf " ${CYAN}%-25s${GREEN}%s${RESET}\n", $$1, $$2} \ + else if (/^## .*$$/) {printf " ${BLUE}%s${RESET}\n", substr($$1,4)} \ + }' $(MAKEFILE_LIST) + @echo \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..4bf8929 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,160 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "strings" +) + +var singleFile bool + +// Function to process a single JSON object and generate an SQL INSERT statement +func processObject(data map[string]interface{}, dbName string) string { + // Generate SQL INSERT statement + columns := []string{} + values := []string{} + + for key, value := range data { + switch v := value.(type) { + case nil: + continue + case string: + // Escape single quotes in the string + escapedValue := strings.ReplaceAll(v, "'", "''") + values = append(values, fmt.Sprintf("'%s'", escapedValue)) + case []interface{}: + arrValues := []string{} + for _, item := range v { + arrValues = append(arrValues, fmt.Sprintf("\"%v\"", item)) + } + values = append(values, fmt.Sprintf("'{%s}'", strings.Join(arrValues, ","))) + case float64: + values = append(values, fmt.Sprintf("%v", int(value.(float64)))) + default: + values = append(values, fmt.Sprintf("%v", v)) + } + + columns = append(columns, key) + } + + insertStatement := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", + dbName, + strings.Join(columns, ", "), + strings.Join(values, ", "), + ) + + return insertStatement +} + +// Function to process a single JSON file and generate SQL INSERT statements +func processFile(filePath, dbName string) ([]string, error) { + // Read file content + byteValue, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + + // Parse JSON data + var dataArray []map[string]interface{} + err = json.Unmarshal(byteValue, &dataArray) + if err != nil { + return nil, err + } + + // Generate SQL INSERT statements for each object in the array + var insertStatements []string + for _, data := range dataArray { + insertStatement := processObject(data, dbName) + insertStatements = append(insertStatements, insertStatement) + } + + return insertStatements, nil +} + +// Function to append SQL statements to the output file +func appendToFile(outputFilePath string, newSQLStatements []string) error { + // Open the file in append mode, create it if it doesn't exist + file, err := os.OpenFile(outputFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + + // Append each SQL statement to the file + for _, stmt := range newSQLStatements { + _, err := file.WriteString(stmt + "\n") + if err != nil { + return err + } + } + + return nil +} + +func readDir(path, outputDir string) { + // Read all files in the directory + files, err := os.ReadDir(path) + if err != nil { + panic(err) + } + + for _, file := range files { + if file.IsDir() { + readDir(filepath.Join(path, file.Name()), outputDir) + continue + } + + if !strings.HasSuffix(file.Name(), ".json") { + continue + } + + dbName, _ := strings.CutSuffix(file.Name(), ".json") + + // Process each file + filePath := filepath.Join(path, file.Name()) + sqlStatements, err := processFile(filePath, dbName) + if err != nil { + fmt.Printf("Error processing file %s: %v\n", filePath, err) + continue + } + + // Prepare output file path with .sql extension + outputFileName := strings.TrimSuffix(file.Name(), filepath.Ext(file.Name())) + ".sql" + if singleFile { + outputFileName = "hawapi.sql" + } + + outputFilePath := filepath.Join(outputDir, outputFileName) + + // Append SQL statements to output SQL file + err = appendToFile(outputFilePath, sqlStatements) + if err != nil { + fmt.Printf("Error appending SQL statements to file %s: %v\n", outputFilePath, err) + continue + } + + fmt.Printf("Processed file %s and appended SQL statements to %s\n", filePath, outputFilePath) + } +} + +func main() { + flag.BoolVar(&singleFile, "single", true, "") + flag.Parse() + + // Directory containing JSON files + dirPath := "database/v1/data" + outputDir := "database/v1/data/sql" + + // Create the output directory if it does not exist + if _, err := os.Stat(outputDir); os.IsNotExist(err) { + err = os.Mkdir(outputDir, 0755) + if err != nil { + panic(err) + } + } + + readDir(dirPath, outputDir) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..81b8830 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/HawAPI/api-data + +go 1.22.0 From 1c1c06a95258e389e5212bfc6b8d067189f09aec Mon Sep 17 00:00:00 2001 From: Lucas Josino Date: Sat, 15 Jun 2024 21:25:43 -0300 Subject: [PATCH 3/6] fix: typo in overview translations --- database/v1/data/translations/overview_translations.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/v1/data/translations/overview_translations.json b/database/v1/data/translations/overview_translations.json index f1148a5..bf44a0b 100644 --- a/database/v1/data/translations/overview_translations.json +++ b/database/v1/data/translations/overview_translations.json @@ -3,12 +3,12 @@ "title": "Stranger Things", "description": "When a young boy vanishes, a small town uncovers a mystery involving secret experiments, terrifying supernatural forces and one strange little girl.", "language": "en-US", - "overvie_uuid": "90edba4c-4399-4582-847c-833c597b03c1" + "overview_uuid": "90edba4c-4399-4582-847c-833c597b03c1" }, { "title": "Stranger Things", "description": "Quando um garoto desaparece, a cidade toda participa nas buscas. Mas o que encontram são segredos, forças sobrenaturais e uma menina.", "language": "pt-BR", - "overvie_uuid": "90edba4c-4399-4582-847c-833c597b03c1" + "overview_uuid": "90edba4c-4399-4582-847c-833c597b03c1" } ] From 6a49d21b9c6a2d9f3924f3ebdfd914a1c280b9a4 Mon Sep 17 00:00:00 2001 From: Lucas Josino Date: Mon, 1 Jul 2024 21:04:35 -0300 Subject: [PATCH 4/6] refactor: sql generator --- cmd/main.go | 161 +++++--------------------------- internal/converter/converter.go | 153 ++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 140 deletions(-) create mode 100644 internal/converter/converter.go diff --git a/cmd/main.go b/cmd/main.go index 4bf8929..b3ca505 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,152 +1,23 @@ package main import ( - "encoding/json" "flag" "fmt" "os" - "path/filepath" - "strings" -) - -var singleFile bool - -// Function to process a single JSON object and generate an SQL INSERT statement -func processObject(data map[string]interface{}, dbName string) string { - // Generate SQL INSERT statement - columns := []string{} - values := []string{} - - for key, value := range data { - switch v := value.(type) { - case nil: - continue - case string: - // Escape single quotes in the string - escapedValue := strings.ReplaceAll(v, "'", "''") - values = append(values, fmt.Sprintf("'%s'", escapedValue)) - case []interface{}: - arrValues := []string{} - for _, item := range v { - arrValues = append(arrValues, fmt.Sprintf("\"%v\"", item)) - } - values = append(values, fmt.Sprintf("'{%s}'", strings.Join(arrValues, ","))) - case float64: - values = append(values, fmt.Sprintf("%v", int(value.(float64)))) - default: - values = append(values, fmt.Sprintf("%v", v)) - } - - columns = append(columns, key) - } - - insertStatement := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", - dbName, - strings.Join(columns, ", "), - strings.Join(values, ", "), - ) - - return insertStatement -} - -// Function to process a single JSON file and generate SQL INSERT statements -func processFile(filePath, dbName string) ([]string, error) { - // Read file content - byteValue, err := os.ReadFile(filePath) - if err != nil { - return nil, err - } - - // Parse JSON data - var dataArray []map[string]interface{} - err = json.Unmarshal(byteValue, &dataArray) - if err != nil { - return nil, err - } - - // Generate SQL INSERT statements for each object in the array - var insertStatements []string - for _, data := range dataArray { - insertStatement := processObject(data, dbName) - insertStatements = append(insertStatements, insertStatement) - } - - return insertStatements, nil -} - -// Function to append SQL statements to the output file -func appendToFile(outputFilePath string, newSQLStatements []string) error { - // Open the file in append mode, create it if it doesn't exist - file, err := os.OpenFile(outputFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return err - } - defer file.Close() - - // Append each SQL statement to the file - for _, stmt := range newSQLStatements { - _, err := file.WriteString(stmt + "\n") - if err != nil { - return err - } - } - - return nil -} -func readDir(path, outputDir string) { - // Read all files in the directory - files, err := os.ReadDir(path) - if err != nil { - panic(err) - } - - for _, file := range files { - if file.IsDir() { - readDir(filepath.Join(path, file.Name()), outputDir) - continue - } - - if !strings.HasSuffix(file.Name(), ".json") { - continue - } - - dbName, _ := strings.CutSuffix(file.Name(), ".json") - - // Process each file - filePath := filepath.Join(path, file.Name()) - sqlStatements, err := processFile(filePath, dbName) - if err != nil { - fmt.Printf("Error processing file %s: %v\n", filePath, err) - continue - } - - // Prepare output file path with .sql extension - outputFileName := strings.TrimSuffix(file.Name(), filepath.Ext(file.Name())) + ".sql" - if singleFile { - outputFileName = "hawapi.sql" - } - - outputFilePath := filepath.Join(outputDir, outputFileName) - - // Append SQL statements to output SQL file - err = appendToFile(outputFilePath, sqlStatements) - if err != nil { - fmt.Printf("Error appending SQL statements to file %s: %v\n", outputFilePath, err) - continue - } - - fmt.Printf("Processed file %s and appended SQL statements to %s\n", filePath, outputFilePath) - } -} + "github.com/HawAPI/api-data/internal/converter" +) func main() { - flag.BoolVar(&singleFile, "single", true, "") - flag.Parse() + var singleFile bool + var inputDir string + var outputDir string - // Directory containing JSON files - dirPath := "database/v1/data" - outputDir := "database/v1/data/sql" + flag.BoolVar(&singleFile, "single", true, "define if output will be in a single or multiple files") + flag.StringVar(&inputDir, "in", "database/v1/data", "input directory") + flag.StringVar(&outputDir, "out", "database/v1/data/sql", "output directory") + flag.Parse() + flag.Usage = cmdUsage // Create the output directory if it does not exist if _, err := os.Stat(outputDir); os.IsNotExist(err) { @@ -156,5 +27,15 @@ func main() { } } - readDir(dirPath, outputDir) + conv := converter.New(inputDir, outputDir, singleFile) + err := conv.Start() + if err != nil { + panic(err) + } +} + +func cmdUsage() { + w := flag.CommandLine.Output() + fmt.Fprintf(w, "Usage of %s: \n", os.Args[0]) + flag.PrintDefaults() } diff --git a/internal/converter/converter.go b/internal/converter/converter.go new file mode 100644 index 0000000..9a84b34 --- /dev/null +++ b/internal/converter/converter.go @@ -0,0 +1,153 @@ +package converter + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +type Converter struct { + inputDir, outputDir string + singleFile bool +} + +func New(inputDir, outputDir string, singleFile bool) Converter { + return Converter{ + inputDir, + outputDir, + singleFile, + } +} + +func (c *Converter) Start() error { + err := c.processFiles(c.inputDir) + if err != nil { + return err + } + + return nil +} + +// processObject will generate an SQL INSERT statement +func (c *Converter) processObject(data map[string]interface{}, db string) string { + columns := []string{} + values := []string{} + + for key, value := range data { + switch v := value.(type) { + case nil: + continue + case string: + // Escape single quotes in the string + escapedValue := strings.ReplaceAll(v, "'", "''") + values = append(values, fmt.Sprintf("'%s'", escapedValue)) + case []interface{}: + arrValues := []string{} + for _, item := range v { + arrValues = append(arrValues, fmt.Sprintf("\"%v\"", item)) + } + values = append(values, fmt.Sprintf("'{%s}'", strings.Join(arrValues, ","))) + case float64: + values = append(values, fmt.Sprintf("%v", int(value.(float64)))) + default: + values = append(values, fmt.Sprintf("%v", v)) + } + + columns = append(columns, key) + } + + return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", + db, + strings.Join(columns, ", "), + strings.Join(values, ", "), + ) +} + +// processFile will process a single JSON file +func (c *Converter) processFile(filePath, db string) ([]string, error) { + file, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + + var data []map[string]interface{} + err = json.Unmarshal(file, &data) + if err != nil { + return nil, err + } + + var statements []string + for _, ob := range data { + statement := c.processObject(ob, db) + statements = append(statements, statement) + } + + return statements, nil +} + +// appendToFile will append SQL statements to the output file +func (c *Converter) appendToFile(outFilePath string, statements []string) error { + // Open the file in append mode, create it if it doesn't exist + file, err := os.OpenFile(outFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + + for _, stmt := range statements { + _, err := file.WriteString(stmt + "\n") + if err != nil { + return err + } + } + + return nil +} + +func (c *Converter) processFiles(dir string) error { + files, err := os.ReadDir(dir) + if err != nil { + panic(err) + } + + for _, file := range files { + if file.IsDir() { + c.processFiles(filepath.Join(dir, file.Name())) + continue + } + + if !strings.HasSuffix(file.Name(), ".json") { + continue + } + + db, _ := strings.CutSuffix(file.Name(), ".json") + if strings.Contains(db, "overview") { + db = strings.ReplaceAll(db, "overview", "overviews") + } + + filePath := filepath.Join(dir, file.Name()) + sqlStatements, err := c.processFile(filePath, db) + if err != nil { + fmt.Printf("Error processing file %s: %v\n", filePath, err) + continue + } + + outFileName := strings.TrimSuffix(file.Name(), ".json") + ".sql" + if c.singleFile { + outFileName = "hawapi.sql" + } + + outFilePath := filepath.Join(c.outputDir, outFileName) + err = c.appendToFile(outFilePath, sqlStatements) + if err != nil { + fmt.Printf("Error appending SQL statements to file %s: %v\n", outFilePath, err) + continue + } + + fmt.Printf("Processed file %s and appended SQL statements to %s\n", filePath, outFilePath) + } + + return nil +} From 10a594bb49ad0419ed94cb75fd50b42f02f37f25 Mon Sep 17 00:00:00 2001 From: Lucas Josino Date: Mon, 1 Jul 2024 21:05:19 -0300 Subject: [PATCH 5/6] build: add workflow to generate SQL file --- .github/workflows/generate-sql.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/generate-sql.yml diff --git a/.github/workflows/generate-sql.yml b/.github/workflows/generate-sql.yml new file mode 100644 index 0000000..757fd0c --- /dev/null +++ b/.github/workflows/generate-sql.yml @@ -0,0 +1,27 @@ +name: "Generate SQL" + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - "database/**" + +jobs: + generate-sql: + name: Go test with coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Golang + uses: actions/setup-go@v5 + with: + go-version: "1.22" + + - name: Install dependencies + run: go mod tidy + + - name: Run command to generate SQL + run: go run cmd/main.go --single --in database/v1/data --out database/v1/data/sql From bc8490d1510e9206c1e84e149c4416839a068a22 Mon Sep 17 00:00:00 2001 From: Lucas Josino Date: Mon, 1 Jul 2024 21:06:42 -0300 Subject: [PATCH 6/6] chore: remove `*.sql` from gitignore --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index e54761d..c12fadb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,4 @@ node_modules/ dev/ .env -.env.dev - -*.sql \ No newline at end of file +.env.dev \ No newline at end of file