PDF Table to Pandas Data Frame to PDF Map

import pandas as pd
import numpy as np
import requests

url = requests.get("https://www.fire.ca.gov/our-impact/statistics/")
df = pd.read_html(url.text)[1]
df.head()

pdf_url_list = ["https://34c031f8-c9fd-4018-8c5a-4159cdff6b0d-cdn-endpoint.azureedge.net/-/media/calfire-website/our-impact/fire-statistics/featured-items/top20_acres.pdf",
                "https://34c031f8-c9fd-4018-8c5a-4159cdff6b0d-cdn-endpoint.azureedge.net/-/media/calfire-website/our-impact/fire-statistics/featured-items/top20_destruction.pdf",
                "https://34c031f8-c9fd-4018-8c5a-4159cdff6b0d-cdn-endpoint.azureedge.net/-/media/calfire-website/our-impact/fire-statistics/featured-items/top20_deadliest.pdf"]

from tabula import wrapper
df = wrapper.read_pdf(pdf_url_list[0])

# in order to print first 5 lines of Table
df.head()

# new data frame with split value columns 
new = df["FIRE NAME (CAUSE)"].str.split("(", n = 1, expand = True) 
  
df["ID"] = new[0].str[0]

# making separate first name column from new data frame 
df["FIRE NAME"]= new[0].str.replace('\r', '').str[1:]

# making separate last name column from new data frame 
df["CAUSE"]= new[1].str[:-1]

df = df.drop(["FIRE NAME (CAUSE)"], axis=1)

df.drop(df.tail(1).index,inplace=True)

df.head()

import re

"""Used to extract numbers from composite string e.g. "271,911 CA /\r43,666 NV" 
   and add the acres within two or more states
"""
def extract_and_sum(s):
    tmp = map(int, re.sub(r'[ a-zA-Z,]+[ a-zA-Z,]+', '', s, re.I).replace(',','').split('/\r'))
    return sum(list(tmp))

"""Used to turn numerics e.g. 273,911 to int without delimitors e.g. 273911
"""
def replace_and_reassign(s):
    return int(s.replace(',',''))

temp= df['ACRES'].apply(lambda x: extract_and_sum(x) if '\r' in x else replace_and_reassign(x))

df['ACRES'] = temp
df.head()

"""Using `numpy.arrange` in creating new rows as place holder to hold the splitted strings from df[col] based on `sep`;
   then `iloc` and `assign` to place the splitted results into newly added rows
"""
def explode_str(df, col, sep):
    s = df[col]
    i = np.arange(len(s)).repeat(s.str.count(sep) + 1)
    return df.iloc[i].assign(**{col: sep.join(s).split(sep)})

new_df = explode_str(df, 'COUNTY', ',')

new_df = explode_str(new_df, 'COUNTY', '&')

new_df['COUNTY'] = new_df['COUNTY'].str.replace('\r','')

new_df['NAME'] = new_df['COUNTY'].str.replace(' County','')

new_df['STATE'] = 'CA'

new_df.head()

from arcgis.gis import GIS

gis = GIS('home')

counties_item = gis.content.get('48f9af87daa241c4b267c5931ad3b226')
counties_item

counties_flayer = counties_item.layers[0]
counties_fset = counties_flayer.query("STATE_NAME='California'")
counties_fset.sdf['NAME'].head()

0      Alameda County
1       Alpine County
2       Amador County
3        Butte County
4    Calaveras County
Name: NAME, dtype: string

counties_fset.sdf.columns

Index(['FID', 'NAME', 'STATE_NAME', 'STATE_FIPS', 'CNTY_FIPS', 'FIPS',
       'POPULATION', 'POP_SQMI', 'POP2010', 'POP10_SQMI', 'WHITE', 'BLACK',
       'AMERI_ES', 'ASIAN', 'HAWN_PI', 'HISPANIC', 'OTHER', 'MULT_RACE',
       'MALES', 'FEMALES', 'AGE_UNDER5', 'AGE_5_9', 'AGE_10_14', 'AGE_15_19',
       'AGE_20_24', 'AGE_25_34', 'AGE_35_44', 'AGE_45_54', 'AGE_55_64',
       'AGE_65_74', 'AGE_75_84', 'AGE_85_UP', 'MED_AGE', 'MED_AGE_M',
       'MED_AGE_F', 'HOUSEHOLDS', 'AVE_HH_SZ', 'HSEHLD_1_M', 'HSEHLD_1_F',
       'MARHH_CHD', 'MARHH_NO_C', 'MHH_CHILD', 'FHH_CHILD', 'FAMILIES',
       'AVE_FAM_SZ', 'HSE_UNITS', 'VACANT', 'OWNER_OCC', 'RENTER_OCC',
       'NO_FARMS12', 'AVE_SIZE12', 'CROP_ACR12', 'AVE_SALE12', 'SQMI',
       'NO_FARMS17', 'AVE_SIZE17', 'CROP_ACR17', 'AVE_SALE17', 'Shape_Leng',
       'Shape_Area', 'Shape__Area', 'Shape__Length', 'SHAPE'],
      dtype='object')

county_df = counties_fset.sdf

# new data frame with split value columns
county_name = county_df["NAME"].str.split(" ", n = 1, expand = True)
 
# making separate first name column from new data frame
county_df["NAME"]= county_name[0]
 
# Dropping old Name columns
#counties_fset.sdf.drop(columns =["NAME"], inplace = True)

county_df['NAME'].head()

0      Alameda
1       Alpine
2       Amador
3        Butte
4    Calaveras
Name: NAME, dtype: string

cols_2 = ['NAME', 'POPULATION', 'POP_SQMI',
          'SHAPE', 'SQMI', 'STATE_FIPS', 'STATE_NAME', 'Shape_Area',
          'Shape_Leng', 'Shape__Area', 'Shape__Length']
overlap_rows = pd.merge(left = county_df[cols_2], right = new_df, how='inner',
                        on = 'NAME')
overlap_rows.head()

map1 = gis.map('California, USA')
map1.layout.height = '650px'
map1.legend = True
map1

map1.clear_graphics()
overlap_rows.spatial.plot(kind='map', map_widget=map1,
                          renderer_type='c',  # for class breaks renderer
                          method='esriClassifyNaturalBreaks',  # classification algorithm
                          class_count=7,  # choose the number of classes
                          col='ACRES',  # numeric column to classify
                          cmap='hot',  # color map to pick colors from for each class
                          alpha=0.7  # specify opacity
                         )

True

item_properties = {
    "title": "Top 20 largest Wildfires in California Till April 2023",
    "tags" : "wildfire",
    "snippet": "read pdf and export pdf",
    "description": "test description"
}
item = map1.save(item_properties)
item

webmap_json = item.get_data()
webmap_json.keys()

dict_keys(['operationalLayers', 'baseMap', 'spatialReference', 'version', 'authoringApp', 'authoringAppVersion'])

map1.extent

{'spatialReference': {'latestWkid': 3857, 'wkid': 102100},
 'xmin': -14383204.412078053,
 'ymin': 3664734.601863475,
 'xmax': -11976355.265435062,
 'ymax': 5254624.790194718}

webmap_json['mapOptions'] = {
    "extent" : {
        "xmin": -14383204.412078043,
        "ymin": 3664734.6018634685,
        "xmax": -11976355.2654350533,
        "ymax": 5254624.790194712,
        "spatialReference": {
            "latestWkid": 3857,
            "wkid": 102100
        }
    },
    "scale" : 9244648.868618,
    "spatialReference" : {
        "wkid" : 102100
    }
}

webmap_json['exportOptions'] = { "dpi" : 96,
                                 "outputSize" :  [746, 575]
                               }

from arcgis.mapping import export_map
res = export_map(web_map_as_json=webmap_json, format='PDF', layout_template='MAP_ONLY')
#res["url"]
res.url

'https://utility.arcgisonline.com/arcgis/rest/directories/arcgisoutput/Utilities/PrintingTools_GPServer/x_____xl3SfXO9Y8JofGGdEpHmfWw..x_____x_ags_7bdbaffa-e78b-11ea-96b8-22000bb3a270.pdf'