scripts/02_adjust_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "# Introdução\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pprint\n",
    "import shutil\n",
    "import tarfile\n",
    "import tempfile\n",
    "from pathlib import Path\n",
    "\n",
    "import geopandas as gpd\n",
    "import pandas as pd\n",
    "from paths import input_path, input_path_down, output_path_gpkg\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "## List Files\n",
    "\n",
    "Lista todos os arquivos que foram feitos download.\n",
    "\n",
    "São arquivos em formato .tar, por município!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lista Downloads\n",
    "list_tar_files = list(input_path_down.rglob('*.tar'))\n",
    "list_tar_files = list(set(list_tar_files))\n",
    "list_tar_files.sort()\n",
    "\n",
    "# Results\n",
    "print(f'N Pastas {len(list_tar_files)}')\n",
    "pprint.pprint(list_tar_files[:5])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "Após isso criei uma lista com todos os arquivos .shp que estão dentro dos arquivos .tar.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_shape_files = []\n",
    "\n",
    "for tar_file in list_tar_files:\n",
    "    # print(tar_file)\n",
    "    # list_shp = []\n",
    "    with tarfile.open(tar_file, 'r') as tar_obj:\n",
    "        for member in tar_obj.getmembers():\n",
    "            extension = Path(member.name).suffix\n",
    "            if extension == '.shp':\n",
    "                # print(member)\n",
    "                name = member.get_info()['name']\n",
    "                # print(name)\n",
    "                tar_file_virtual = f'tar://{tar_file.as_posix()}/{name}'\n",
    "                # print(tar_file_virtual)\n",
    "                list_shape_files.append(tar_file_virtual)\n",
    "\n",
    "# Results\n",
    "print(f'São {len(list_shape_files)} shapefiles')\n",
    "pprint.pprint(list_shape_files[:5])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "Criei função que converte cada item da célula em uma diversidade de colunas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_infos(row):\n",
    "    \"\"\"\n",
    "    _summary_\n",
    "\n",
    "    :param my_url: _description_\n",
    "    :type my_url: _type_\n",
    "    :return: _description_\n",
    "    :rtype: _type_\n",
    "    \"\"\"\n",
    "    my_url = row['url']\n",
    "\n",
    "    url_path = Path(my_url.replace('tar://', ''))\n",
    "    url_path_part = url_path.relative_to(input_path_down)\n",
    "    parts = url_path_part.parts[0:-1]\n",
    "    return (\n",
    "        url_path,\n",
    "        url_path.relative_to(input_path_down),\n",
    "        url_path_part.parts[0],\n",
    "        url_path_part.parts[1],\n",
    "        url_path_part.parts[2],\n",
    "        url_path_part.name,\n",
    "    )\n",
    "\n",
    "\n",
    "def find_file_type(name):\n",
    "    return name.split('_', maxsplit=2)[2]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Dataframe\n",
    "df = pd.DataFrame(list_shape_files, columns=['url'])\n",
    "\n",
    "# Create Columns\n",
    "df[\n",
    "    ['path', 'path_tar', 'nome_tarfile', 'sub_1', 'sub_2', 'nome_shape']\n",
    "] = df.apply(\n",
    "    find_infos,\n",
    "    result_type='expand',\n",
    "    axis=1,\n",
    ")\n",
    "df['nome_tipo'] = df['nome_shape'].apply(find_file_type)\n",
    "\n",
    "# Results\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tipos das Subbacias\n",
    "list_sub_2 = list(set(df['sub_2']))\n",
    "list_sub_2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tipos de Shapefiles\n",
    "list_nome_tipo = list(set(df['nome_tipo']))\n",
    "list_nome_tipo.sort()\n",
    "list_nome_tipo\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "## Temporários\n",
    "\n",
    "Como não quero extrair os arquivos .tar, criei centenas de arquivos temporários.\n",
    "\n",
    "... para que depois eu pudesse \"copiar\" a estrutura dal lista de arquivos.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clear TempFiles Folder\n",
    "input_path_tempfiles = input_path / 'temp_files'\n",
    "input_path_tempfiles.mkdir(exist_ok=True)\n",
    "shutil.rmtree(input_path_tempfiles)\n",
    "input_path_tempfiles.mkdir(exist_ok=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp = df[0:5]\n",
    "df_temp = df\n",
    "for index, row in df_temp.iterrows():\n",
    "    with tempfile.NamedTemporaryFile(\n",
    "        dir=input_path_tempfiles, delete=False\n",
    "    ) as tmp:\n",
    "        # print(tmp.name)\n",
    "        tmp.close()\n",
    "\n",
    "    # Parts\n",
    "    parts = Path(row['path_tar']).parts[1:]\n",
    "    my_path = parts[:-1]\n",
    "    filename = parts[-1]\n",
    "\n",
    "    # Path\n",
    "    new_filepath = input_path_tempfiles / Path(*my_path)\n",
    "    new_filepath.mkdir(exist_ok=True, parents=True)\n",
    "    print(new_filepath)\n",
    "\n",
    "    # Rename\n",
    "    shutil.move(tmp.name, new_filepath / filename)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_nome_tipo\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# APP\n",
    "list_files_app = list(input_path_tempfiles.rglob('*APP/*APP.shp'))\n",
    "list_files_app_uso = list(input_path_tempfiles.rglob('*APP/*APP_USO.shp'))\n",
    "\n",
    "# HIDROGRAFIA\n",
    "list_files_hidro_simples = list(\n",
    "    input_path_tempfiles.rglob('*HIDROGRAFIA/*RIOS_SIMPLES.shp')\n",
    ")\n",
    "list_files_hidro_duplas = list(\n",
    "    input_path_tempfiles.rglob('*HIDROGRAFIA/*RIOS_DUPLOS*.shp')\n",
    ")\n",
    "list_files_hidro_nascentes = list(\n",
    "    input_path_tempfiles.rglob('*HIDROGRAFIA/*NASCENTE*.shp')\n",
    ")\n",
    "list_files_hidro_massa = list(\n",
    "    input_path_tempfiles.rglob('*HIDROGRAFIA/*MASSA*_DAGUA*.shp')\n",
    ")\n",
    "\n",
    "# USO\n",
    "list_files_uso = list(input_path_tempfiles.rglob('*USO/*USO*.shp'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "## Teste de ler Gpd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_choose = list_files_hidro_massa\n",
    "list_shps = [f.relative_to(input_path_tempfiles) for f in list_choose]\n",
    "shps = list_shps[1]\n",
    "shps.parts[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tar_file = input_path_down / f'{shps.parts[0]}.tar' / shps\n",
    "tar_file = f'tar://{tar_file.as_posix()}'\n",
    "tar_file\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf = gpd.read_file(tar_file)\n",
    "gdf.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "## Read Data and Concat\n",
    "\n",
    "- gdal.SetConfigOption('SHAPE_RESTORE_SHX', 'YES')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_lists = {\n",
    "    'app': list_files_app,\n",
    "    'app_uso': list_files_app_uso,\n",
    "    'hidro_simples': list_files_hidro_simples,\n",
    "    'hidro_duplas': list_files_hidro_duplas,\n",
    "    'hidro_nascentes': list_files_hidro_nascentes,\n",
    "    'hidro_massa': list_files_hidro_massa,\n",
    "    'uso': list_files_uso,\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k, v in dict_lists.items():\n",
    "     print(k)\n",
    "     \n",
    "     # Etapa 1: Avalia se é possível ver os arquivos shp dentro do TAR\n",
    "     list_fix = []\n",
    "     list_choose = list(v)\n",
    "     for shp in list_choose:\n",
    "          # Paths\n",
    "          my_path = shp.relative_to(input_path_tempfiles)\n",
    "          tar_file = input_path_down / f'{my_path.parts[0]}.tar' / my_path\n",
    "          tar_file = f'tar://{tar_file.as_posix()}'\n",
    "          list_fix.append(tar_file)\n",
    "          try:\n",
    "               gdf = gpd.read_file(tar_file)\n",
    "\n",
    "          except Exception as e:\n",
    "               print(tar_file)\n",
    "               print(e)\n",
    "               pass\n",
    "\n",
    "     # Etapa 2: Junta tudo!\n",
    "     list_shp = [gpd.read_file(shp).to_crs(epsg=4326) for shp in list_fix]\n",
    "     gdf = gpd.GeoDataFrame(pd.concat(list_shp, ignore_index=True), crs=4326)\n",
    "     \n",
    "     # Etapa 3: Save\n",
    "     gdf.to_file(\n",
    "          output_path_gpkg / f'{k}.gpkg',\n",
    "          layer=f'{k}',\n",
    "          driver='GPKG',\n",
    "          OVERWRITE=True\n",
    "     )\n",
    "     print(f'Fim do {k}')\n",
    "\n",
    "# End\n",
    "print('Fim Geral')\n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "shutil.rmtree(input_path_tempfiles)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pablocarreira-lastest",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8 | packaged by conda-forge | (main, Nov 22 2022, 08:16:33) [MSC v.1929 64 bit (AMD64)]"
  },
  "vscode": {
   "interpreter": {
    "hash": "3eff2e4eec0fbea9d660dbbf3c9c38e7f3540f855f42d9d14e44150e78f59c2a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}