Data cleansing of open data of the occurrence situation of the Ministry of Health, Labor and Welfare

Ministry of Health, Labor and Welfare, domestic outbreak situation, etc. Open data

Data cleansing because csv contains \ n newline characters and the previous day's ratio is in the same cell

import re
import pandas as pd

df = pd.read_csv("https://www.mhlw.go.jp/content/current_situation.csv", index_col=0)

df.index = df.index.str.replace(r"※\d", "").str.replace(",", "").str.replace(r"\\n", "")
df.columns = df.columns.str.replace(r"※\d", "").str.replace(r"\\n", "").str.strip()

df = df.applymap(lambda s: re.sub(r"※\d", "", s))

dfs = []

for name, col in df.iteritems():

    df_tmp = col.str.split(r"\\n", expand=True).rename(columns={0: "Cumulative", 1: "The day before ratio"})
    df_tmp.columns = pd.MultiIndex.from_product([[name], df_tmp.columns])

    dfs.append(df_tmp)

df = pd.concat(dfs, axis=1).fillna(0)

df = df.applymap(lambda s: str(s).replace(",", "").strip().strip("()")).astype(int)

df.to_csv("current_situation.csv", encoding="utf_8_sig")