Automatically determine and process the encoding of the text file

def conv_encoding(data):
    lookup = ('utf_8', 'euc_jp', 'euc_jis_2004', 'euc_jisx0213',
            'shift_jis', 'shift_jis_2004','shift_jisx0213',
            'iso2022jp', 'iso2022_jp_1', 'iso2022_jp_2', 'iso2022_jp_3',
            'iso2022_jp_ext','latin_1', 'ascii')
    encode = None
    for encoding in lookup:
      try:
        data = data.decode(encoding)
        encode = encoding
        break
      except:
        pass
    if isinstance(data, unicode):
        return data,encode
    else:
        raise LookupError

#File reading and encoding investigation
fp = open(path,'r')
str,encoding = None,None
try:
  str,encoding = conv_encoding(fp.read())
finally:
  fp.close()

#Edit content
...<Arbitrary code>


#Write file in original encoding
fp = open(path,'w')
try:
  fp.write(str.encode(encoding))
finally:
  fp.close()

Recommended Posts