Document Actions

Unicode to ISCII

Module that provides Unicode to ISCII conversion functions. Requires Mary’s iscii2utf8.py script.

Size 8.5 kB - File type text/python-source

File contents

#!/usr/bin/env python
#######################################################################
#
# Unicode to ISCII convertor script. Version 0.2
#
# Copyright (C) 2004, Kiran Jonnalagadda. All rights reserved.
# May be used under the terms of the BSD license. Please send your bug
# reports and suggestions to jace at pobox dot com or jace at seacrow
# dot com. http://jace.seacrow.com/
#
# Inspired by Meyarivan's iscii2utf8 script and the Python Unicode
# framework. Written for Mahiti. http://www.mahiti.org/
#
# Usage:
#
#   Command line: [TODO]
#     unicode2iscii unicode-text.txt ... (blank for STDIN)
#
#   Python (option 1):
#     from unicode2iscii import unicode2iscii, iscii_languages
#     # unicode_str here must be a Python Unicode string
#     iscii_str = unicode2iscii(unicode_str)
#     # iscii_scripts returns a list of scripts found in the string
#     languages = iscii_languages(unicode_str)
#
#   Python (option 2): [TODO]
#     import unicode2iscii
#     # This also requires a Python Unicode string
#     iscii_str = unicode_str.encode('iscii')
#
#######################################################################
#
# ChangeLog:
#
#   0.2 (2004/03/29):
#     * Second release. Now tested to handle Devanagari text correctly.
#       Still not tested with other scripts.
#     * Python Option 1 in the API works. CLI and Unicode codec API not
#       implemented yet.
#
#   0.1 (2004/03/24):
#     * First test release. Only the iscii_scripts function works
#       correctly.
#     * The unicode2iscii function only does codepoint manipulation.
#       Special cases are not handled yet.
#
#######################################################################

from StringIO import StringIO

# List of ISCII scripts

ISCII_DEVANAGARI = 0
ISCII_BENGALI    = 1
ISCII_ASSAMESE   = 1 # Same as Bengali; we won't bother to distinguish.
ISCII_GURMUKHI   = 2
ISCII_GUJARATI   = 3
ISCII_ORIYA      = 4
ISCII_TAMIL      = 5
ISCII_TELUGU     = 6
ISCII_KANNADA    = 7
ISCII_MALAYALAM  = 8

# Common constants

ISCII_START        = 0xA0
ISCII_HALANT       = 0xE8
ISCII_NUKTA        = 0xE9
ISCII_VIRAM        = 0xEA
ISCII_INV          = 0xD9
UNICODE_DEVANAGARI = 0x0900
UNICODE_ZWNJ       = 0x200C
UNICODE_ZWJ        = 0x200D
UNICODE_CGJ        = 0x034F

# Table of ISCII script names in English.

iscii_script_names = {
    ISCII_DEVANAGARI: 'Devanagari',
    ISCII_BENGALI: 'Bengali',
    ISCII_GURMUKHI: 'Gurmukhi',
    ISCII_GUJARATI: 'Gujarati',
    ISCII_ORIYA: 'Oriya',
    ISCII_TAMIL: 'Tamil',
    ISCII_TELUGU: 'Telugu',
    ISCII_KANNADA: 'Kannada',
    ISCII_MALAYALAM: 'Malayalam',
    }

# Table of mappings from Unicode character range to ISCII script

unicode_iscii_scripts = {
    (0x0900, 0x097F): ISCII_DEVANAGARI,
    (0x0980, 0x09FF): ISCII_BENGALI,
    (0x0A00, 0x0A7F): ISCII_GURMUKHI,
    (0x0A80, 0x0AFF): ISCII_GUJARATI,
    (0x0B00, 0x0B7F): ISCII_ORIYA,
    (0x0B80, 0x0BFF): ISCII_TAMIL,
    (0x0C00, 0x0C7F): ISCII_TELUGU,
    (0x0C80, 0x0CFF): ISCII_KANNADA,
    (0x0D00, 0x0D7F): ISCII_MALAYALAM,
    }

# Build a faster script lookup table

fast_unicode_iscii_scripts = {}

for key in unicode_iscii_scripts:
    for fastkey in range(key[0], key[1]+1):
        fast_unicode_iscii_scripts[fastkey] = unicode_iscii_scripts[key]

# Build an offset conversion map for each script. ISCII characters start
# at 0xA0. Unicode uses the same sequence of characters as ISCII, but
# with different offsets and with a few exceptions.

unicode_offsets = {}

for key, value in unicode_iscii_scripts.iteritems():
    unicode_offsets[value] = key[0] - UNICODE_DEVANAGARI

# Reverse the iscii_to_unicode table from Mary's script.

from iscii2utf8 import iscii_to_unicode

unicode_to_iscii = {}

for key, value in iscii_to_unicode.iteritems():
    if value != 0xFFFF:
        unicode_to_iscii[value] = (key,)

unicode_to_iscii.update({
    0x0958: (0xB3, ISCII_NUKTA), # ka
    0x0959: (0xB4, ISCII_NUKTA), # Ka
    0x095A: (0xB5, ISCII_NUKTA), # ga
    0x095B: (0xBA, ISCII_NUKTA), # ja
    0x095C: (0xBF, ISCII_NUKTA), # da
    0x095D: (0xC0, ISCII_NUKTA), # Da
    0x095E: (0xC9, ISCII_NUKTA), # fa
    0x0965: (ISCII_VIRAM, ISCII_VIRAM), # Double viram (full stop)
    0x093D: (ISCII_VIRAM, ISCII_NUKTA), # Avagraha
    })

# Public functions ----------------------------------------------------

def iscii_scripts(ustr):
    """
    Takes a unicode string as parameter and returns all the ISCII
    scripts found in the string.
    """
    scripts = {}
    for char in ustr:
        scripts[fast_unicode_iscii_scripts.get(ord(char), None)] = None
    result = scripts.keys()
    try: result.remove(None)
    except ValueError: pass
    return result

def unicode2iscii(ustr, unknownord=255):
    """
    Encodes the given Unicode string as ISCII. Raises ValueError if the
    string contains more than one ISCII script. All unknown Unicode
    characters are replaced with ordinal 255 or with the value of
    optional parameter unknownord (should be an integer 0..255).
    """
    scripts = iscii_scripts(ustr)
    if len(scripts) > 1:
        raise ValueError, "Multiple ISCII scripts in string."
    elif len(scripts) == 0:
        script = None
    else:
        script = scripts[0]

# We won't set default values for lbound and ubound because that
    # should never be needed.
    if script is not None:
        for key, value in unicode_iscii_scripts.iteritems():
            if value == script:
                lbound = key[0]
                ubound = key[1]
                break

offset = unicode_offsets.get(script, None)

stream = StringIO(ustr)

iscii_str = []
    while stream.pos < stream.len:
        charval = ord(stream.read(1))
        # Special cases are handled here.
        if charval < ISCII_START: # Character is below the ISCII range.
            iscii_str.append(chr(charval))
        elif charval == UNICODE_ZWNJ: # Explicit halant
            iscii_str.append(chr(ISCII_HALANT))
        elif charval == UNICODE_ZWJ: # Soft halant (consonant half-form)
            iscii_str.append(chr(ISCII_NUKTA))
        elif charval == UNICODE_CGJ: # Invisible consonant
            iscii_str.append(chr(ISCII_INV))
        else:
            chars = unicode_to_iscii.get(charval-offset, (unknownord,))
            for char in chars:
                iscii_str.append(chr(char))

return ''.join(iscii_str)

# Internal test functions ---------------------------------------------

def _script_names(list):
    scripts = []
    for item in list:
        scripts.append(iscii_script_names[item])
    return ', '.join(scripts)

def test_script_identify():
    # Test function. Requires the utf8-[1234].txt files.
    print "Test 1: Identifying scripts used in test files."

u0 = u'This is a plain ASCII string.'
    u1 = unicode(file('utf8-1.txt').read(), 'utf-8')
    u2 = unicode(file('utf8-2.txt').read(), 'utf-8')
    u3 = unicode(file('utf8-3.txt').read(), 'utf-8')
    u4 = unicode(file('utf8-4.txt').read(), 'utf-8')

print "ASCII:", _script_names(iscii_scripts(u0))
    print "utf8-1.txt:", _script_names(iscii_scripts(u1))
    print "utf8-2.txt:", _script_names(iscii_scripts(u2))
    print "utf8-3.txt:", _script_names(iscii_scripts(u3))
    print "utf8-4.txt:", _script_names(iscii_scripts(u4))
    print

def test_iscii_encode():
    # Test function. Requires the utf8-[1-4].txt files. Writes to the
    # iscii-[1-4].txt files.
    print "Test 2a: Converting test files from UTF-8 to ISCII."

for filenum in range(1, 5):
        try:
            infile = 'utf8-%d.txt' % filenum
            outfile = 'iscii-%d.txt' % filenum
            out2 = 'iscii-%d.html' % filenum
            ustr = unicode(file(infile).read(), 'utf-8')
            print "%s:" % infile, _script_names(iscii_scripts(ustr))
            print "%s:" % infile,
            for uchar in ustr:
                print hex(ord(uchar)),
            print
            print "Writing to %s..." % outfile,
            file(outfile, 'w').write(unicode2iscii(ustr, ord('?')))
            print "done."
        except ValueError, msg:
            print "error: %s" % msg
    print
    return # End test here. 2b takes too long and works anyway.
    print "Test 2b: Converting a large sample file to ISCII."

infile = "sample.txt"
    outfile = "sample-iscii.txt"
    ustr = unicode(file(infile).read(), 'utf-8')
    print "%s:" % infile, _script_names(iscii_scripts(ustr))
    print "Writing to %s..." % outfile,
    file(outfile, 'w').write(unicode2iscii(ustr, ord('?')))
    print "done."
    print

# Standalone code -----------------------------------------------------

if __name__ == '__main__':
    test_script_identify()
    test_iscii_encode()

tags: python, iscii, unicode