Annotation of embedaddon/libxml2/doc/index.py, revision 1.1
1.1 ! misho 1: #!/usr/bin/python -u
! 2: #
! 3: # imports the API description and fills up a database with
! 4: # name relevance to modules, functions or web pages
! 5: #
! 6: # Operation needed:
! 7: # =================
! 8: #
! 9: # install mysqld, the python wrappers for mysql and libxml2, start mysqld
! 10: # Change the root passwd of mysql:
! 11: # mysqladmin -u root password new_password
! 12: # Create the new database xmlsoft
! 13: # mysqladmin -p create xmlsoft
! 14: # Create a database user 'veillard' and give him passord access
! 15: # change veillard and abcde with the right user name and passwd
! 16: # mysql -p
! 17: # password:
! 18: # mysql> GRANT ALL PRIVILEGES ON xmlsoft TO veillard@localhost
! 19: # IDENTIFIED BY 'abcde' WITH GRANT OPTION;
! 20: #
! 21: # As the user check the access:
! 22: # mysql -p xmlsoft
! 23: # Enter password:
! 24: # Welcome to the MySQL monitor....
! 25: # mysql> use xmlsoft
! 26: # Database changed
! 27: # mysql> quit
! 28: # Bye
! 29: #
! 30: # Then run the script in the doc subdir, it will create the symbols and
! 31: # word tables and populate them with informations extracted from
! 32: # the libxml2-api.xml API description, and make then accessible read-only
! 33: # by nobody@loaclhost the user expected to be Apache's one
! 34: #
! 35: # On the Apache configuration, make sure you have php support enabled
! 36: #
! 37:
! 38: import MySQLdb
! 39: import libxml2
! 40: import sys
! 41: import string
! 42: import os
! 43:
! 44: #
! 45: # We are not interested in parsing errors here
! 46: #
! 47: def callback(ctx, str):
! 48: return
! 49: libxml2.registerErrorHandler(callback, None)
! 50:
! 51: #
! 52: # The dictionnary of tables required and the SQL command needed
! 53: # to create them
! 54: #
! 55: TABLES={
! 56: "symbols" : """CREATE TABLE symbols (
! 57: name varchar(255) BINARY NOT NULL,
! 58: module varchar(255) BINARY NOT NULL,
! 59: type varchar(25) NOT NULL,
! 60: descr varchar(255),
! 61: UNIQUE KEY name (name),
! 62: KEY module (module))""",
! 63: "words" : """CREATE TABLE words (
! 64: name varchar(50) BINARY NOT NULL,
! 65: symbol varchar(255) BINARY NOT NULL,
! 66: relevance int,
! 67: KEY name (name),
! 68: KEY symbol (symbol),
! 69: UNIQUE KEY ID (name, symbol))""",
! 70: "wordsHTML" : """CREATE TABLE wordsHTML (
! 71: name varchar(50) BINARY NOT NULL,
! 72: resource varchar(255) BINARY NOT NULL,
! 73: section varchar(255),
! 74: id varchar(50),
! 75: relevance int,
! 76: KEY name (name),
! 77: KEY resource (resource),
! 78: UNIQUE KEY ref (name, resource))""",
! 79: "wordsArchive" : """CREATE TABLE wordsArchive (
! 80: name varchar(50) BINARY NOT NULL,
! 81: ID int(11) NOT NULL,
! 82: relevance int,
! 83: KEY name (name),
! 84: UNIQUE KEY ref (name, ID))""",
! 85: "pages" : """CREATE TABLE pages (
! 86: resource varchar(255) BINARY NOT NULL,
! 87: title varchar(255) BINARY NOT NULL,
! 88: UNIQUE KEY name (resource))""",
! 89: "archives" : """CREATE TABLE archives (
! 90: ID int(11) NOT NULL auto_increment,
! 91: resource varchar(255) BINARY NOT NULL,
! 92: title varchar(255) BINARY NOT NULL,
! 93: UNIQUE KEY id (ID,resource(255)),
! 94: INDEX (ID),
! 95: INDEX (resource))""",
! 96: "Queries" : """CREATE TABLE Queries (
! 97: ID int(11) NOT NULL auto_increment,
! 98: Value varchar(50) NOT NULL,
! 99: Count int(11) NOT NULL,
! 100: UNIQUE KEY id (ID,Value(35)),
! 101: INDEX (ID))""",
! 102: "AllQueries" : """CREATE TABLE AllQueries (
! 103: ID int(11) NOT NULL auto_increment,
! 104: Value varchar(50) NOT NULL,
! 105: Count int(11) NOT NULL,
! 106: UNIQUE KEY id (ID,Value(35)),
! 107: INDEX (ID))""",
! 108: }
! 109:
! 110: #
! 111: # The XML API description file to parse
! 112: #
! 113: API="libxml2-api.xml"
! 114: DB=None
! 115:
! 116: #########################################################################
! 117: # #
! 118: # MySQL database interfaces #
! 119: # #
! 120: #########################################################################
! 121: def createTable(db, name):
! 122: global TABLES
! 123:
! 124: if db == None:
! 125: return -1
! 126: if name == None:
! 127: return -1
! 128: c = db.cursor()
! 129:
! 130: ret = c.execute("DROP TABLE IF EXISTS %s" % (name))
! 131: if ret == 1:
! 132: print "Removed table %s" % (name)
! 133: print "Creating table %s" % (name)
! 134: try:
! 135: ret = c.execute(TABLES[name])
! 136: except:
! 137: print "Failed to create table %s" % (name)
! 138: return -1
! 139: return ret
! 140:
! 141: def checkTables(db, verbose = 1):
! 142: global TABLES
! 143:
! 144: if db == None:
! 145: return -1
! 146: c = db.cursor()
! 147: nbtables = c.execute("show tables")
! 148: if verbose:
! 149: print "Found %d tables" % (nbtables)
! 150: tables = {}
! 151: i = 0
! 152: while i < nbtables:
! 153: l = c.fetchone()
! 154: name = l[0]
! 155: tables[name] = {}
! 156: i = i + 1
! 157:
! 158: for table in TABLES.keys():
! 159: if not tables.has_key(table):
! 160: print "table %s missing" % (table)
! 161: createTable(db, table)
! 162: try:
! 163: ret = c.execute("SELECT count(*) from %s" % table);
! 164: row = c.fetchone()
! 165: if verbose:
! 166: print "Table %s contains %d records" % (table, row[0])
! 167: except:
! 168: print "Troubles with table %s : repairing" % (table)
! 169: ret = c.execute("repair table %s" % table);
! 170: print "repairing returned %d" % (ret)
! 171: ret = c.execute("SELECT count(*) from %s" % table);
! 172: row = c.fetchone()
! 173: print "Table %s contains %d records" % (table, row[0])
! 174: if verbose:
! 175: print "checkTables finished"
! 176:
! 177: # make sure apache can access the tables read-only
! 178: try:
! 179: ret = c.execute("GRANT SELECT ON xmlsoft.* TO nobody@localhost")
! 180: ret = c.execute("GRANT INSERT,SELECT,UPDATE ON xmlsoft.Queries TO nobody@localhost")
! 181: except:
! 182: pass
! 183: return 0
! 184:
! 185: def openMySQL(db="xmlsoft", passwd=None, verbose = 1):
! 186: global DB
! 187:
! 188: if passwd == None:
! 189: try:
! 190: passwd = os.environ["MySQL_PASS"]
! 191: except:
! 192: print "No password available, set environment MySQL_PASS"
! 193: sys.exit(1)
! 194:
! 195: DB = MySQLdb.connect(passwd=passwd, db=db)
! 196: if DB == None:
! 197: return -1
! 198: ret = checkTables(DB, verbose)
! 199: return ret
! 200:
! 201: def updateWord(name, symbol, relevance):
! 202: global DB
! 203:
! 204: if DB == None:
! 205: openMySQL()
! 206: if DB == None:
! 207: return -1
! 208: if name == None:
! 209: return -1
! 210: if symbol == None:
! 211: return -1
! 212:
! 213: c = DB.cursor()
! 214: try:
! 215: ret = c.execute(
! 216: """INSERT INTO words (name, symbol, relevance) VALUES ('%s','%s', %d)""" %
! 217: (name, symbol, relevance))
! 218: except:
! 219: try:
! 220: ret = c.execute(
! 221: """UPDATE words SET relevance = %d where name = '%s' and symbol = '%s'""" %
! 222: (relevance, name, symbol))
! 223: except:
! 224: print "Update word (%s, %s, %s) failed command" % (name, symbol, relevance)
! 225: print "UPDATE words SET relevance = %d where name = '%s' and symbol = '%s'" % (relevance, name, symbol)
! 226: print sys.exc_type, sys.exc_value
! 227: return -1
! 228:
! 229: return ret
! 230:
! 231: def updateSymbol(name, module, type, desc):
! 232: global DB
! 233:
! 234: updateWord(name, name, 50)
! 235: if DB == None:
! 236: openMySQL()
! 237: if DB == None:
! 238: return -1
! 239: if name == None:
! 240: return -1
! 241: if module == None:
! 242: return -1
! 243: if type == None:
! 244: return -1
! 245:
! 246: try:
! 247: desc = string.replace(desc, "'", " ")
! 248: l = string.split(desc, ".")
! 249: desc = l[0]
! 250: desc = desc[0:99]
! 251: except:
! 252: desc = ""
! 253:
! 254: c = DB.cursor()
! 255: try:
! 256: ret = c.execute(
! 257: """INSERT INTO symbols (name, module, type, descr) VALUES ('%s','%s', '%s', '%s')""" %
! 258: (name, module, type, desc))
! 259: except:
! 260: try:
! 261: ret = c.execute(
! 262: """UPDATE symbols SET module='%s', type='%s', descr='%s' where name='%s'""" %
! 263: (module, type, desc, name))
! 264: except:
! 265: print "Update symbol (%s, %s, %s) failed command" % (name, module, type)
! 266: print """UPDATE symbols SET module='%s', type='%s', descr='%s' where name='%s'""" % (module, type, desc, name)
! 267: print sys.exc_type, sys.exc_value
! 268: return -1
! 269:
! 270: return ret
! 271:
! 272: def addFunction(name, module, desc = ""):
! 273: return updateSymbol(name, module, 'function', desc)
! 274:
! 275: def addMacro(name, module, desc = ""):
! 276: return updateSymbol(name, module, 'macro', desc)
! 277:
! 278: def addEnum(name, module, desc = ""):
! 279: return updateSymbol(name, module, 'enum', desc)
! 280:
! 281: def addStruct(name, module, desc = ""):
! 282: return updateSymbol(name, module, 'struct', desc)
! 283:
! 284: def addConst(name, module, desc = ""):
! 285: return updateSymbol(name, module, 'const', desc)
! 286:
! 287: def addType(name, module, desc = ""):
! 288: return updateSymbol(name, module, 'type', desc)
! 289:
! 290: def addFunctype(name, module, desc = ""):
! 291: return updateSymbol(name, module, 'functype', desc)
! 292:
! 293: def addPage(resource, title):
! 294: global DB
! 295:
! 296: if DB == None:
! 297: openMySQL()
! 298: if DB == None:
! 299: return -1
! 300: if resource == None:
! 301: return -1
! 302:
! 303: c = DB.cursor()
! 304: try:
! 305: ret = c.execute(
! 306: """INSERT INTO pages (resource, title) VALUES ('%s','%s')""" %
! 307: (resource, title))
! 308: except:
! 309: try:
! 310: ret = c.execute(
! 311: """UPDATE pages SET title='%s' WHERE resource='%s'""" %
! 312: (title, resource))
! 313: except:
! 314: print "Update symbol (%s, %s, %s) failed command" % (name, module, type)
! 315: print """UPDATE pages SET title='%s' WHERE resource='%s'""" % (title, resource)
! 316: print sys.exc_type, sys.exc_value
! 317: return -1
! 318:
! 319: return ret
! 320:
! 321: def updateWordHTML(name, resource, desc, id, relevance):
! 322: global DB
! 323:
! 324: if DB == None:
! 325: openMySQL()
! 326: if DB == None:
! 327: return -1
! 328: if name == None:
! 329: return -1
! 330: if resource == None:
! 331: return -1
! 332: if id == None:
! 333: id = ""
! 334: if desc == None:
! 335: desc = ""
! 336: else:
! 337: try:
! 338: desc = string.replace(desc, "'", " ")
! 339: desc = desc[0:99]
! 340: except:
! 341: desc = ""
! 342:
! 343: c = DB.cursor()
! 344: try:
! 345: ret = c.execute(
! 346: """INSERT INTO wordsHTML (name, resource, section, id, relevance) VALUES ('%s','%s', '%s', '%s', '%d')""" %
! 347: (name, resource, desc, id, relevance))
! 348: except:
! 349: try:
! 350: ret = c.execute(
! 351: """UPDATE wordsHTML SET section='%s', id='%s', relevance='%d' where name='%s' and resource='%s'""" %
! 352: (desc, id, relevance, name, resource))
! 353: except:
! 354: print "Update symbol (%s, %s, %d) failed command" % (name, resource, relevance)
! 355: print """UPDATE wordsHTML SET section='%s', id='%s', relevance='%d' where name='%s' and resource='%s'""" % (desc, id, relevance, name, resource)
! 356: print sys.exc_type, sys.exc_value
! 357: return -1
! 358:
! 359: return ret
! 360:
! 361: def checkXMLMsgArchive(url):
! 362: global DB
! 363:
! 364: if DB == None:
! 365: openMySQL()
! 366: if DB == None:
! 367: return -1
! 368: if url == None:
! 369: return -1
! 370:
! 371: c = DB.cursor()
! 372: try:
! 373: ret = c.execute(
! 374: """SELECT ID FROM archives WHERE resource='%s'""" % (url))
! 375: row = c.fetchone()
! 376: if row == None:
! 377: return -1
! 378: except:
! 379: return -1
! 380:
! 381: return row[0]
! 382:
! 383: def addXMLMsgArchive(url, title):
! 384: global DB
! 385:
! 386: if DB == None:
! 387: openMySQL()
! 388: if DB == None:
! 389: return -1
! 390: if url == None:
! 391: return -1
! 392: if title == None:
! 393: title = ""
! 394: else:
! 395: title = string.replace(title, "'", " ")
! 396: title = title[0:99]
! 397:
! 398: c = DB.cursor()
! 399: try:
! 400: cmd = """INSERT INTO archives (resource, title) VALUES ('%s','%s')""" % (url, title)
! 401: ret = c.execute(cmd)
! 402: cmd = """SELECT ID FROM archives WHERE resource='%s'""" % (url)
! 403: ret = c.execute(cmd)
! 404: row = c.fetchone()
! 405: if row == None:
! 406: print "addXMLMsgArchive failed to get the ID: %s" % (url)
! 407: return -1
! 408: except:
! 409: print "addXMLMsgArchive failed command: %s" % (cmd)
! 410: return -1
! 411:
! 412: return((int)(row[0]))
! 413:
! 414: def updateWordArchive(name, id, relevance):
! 415: global DB
! 416:
! 417: if DB == None:
! 418: openMySQL()
! 419: if DB == None:
! 420: return -1
! 421: if name == None:
! 422: return -1
! 423: if id == None:
! 424: return -1
! 425:
! 426: c = DB.cursor()
! 427: try:
! 428: ret = c.execute(
! 429: """INSERT INTO wordsArchive (name, id, relevance) VALUES ('%s', '%d', '%d')""" %
! 430: (name, id, relevance))
! 431: except:
! 432: try:
! 433: ret = c.execute(
! 434: """UPDATE wordsArchive SET relevance='%d' where name='%s' and ID='%d'""" %
! 435: (relevance, name, id))
! 436: except:
! 437: print "Update word archive (%s, %d, %d) failed command" % (name, id, relevance)
! 438: print """UPDATE wordsArchive SET relevance='%d' where name='%s' and ID='%d'""" % (relevance, name, id)
! 439: print sys.exc_type, sys.exc_value
! 440: return -1
! 441:
! 442: return ret
! 443:
! 444: #########################################################################
! 445: # #
! 446: # Word dictionnary and analysis routines #
! 447: # #
! 448: #########################################################################
! 449:
! 450: #
! 451: # top 100 english word without the one len < 3 + own set
! 452: #
! 453: dropWords = {
! 454: 'the':0, 'this':0, 'can':0, 'man':0, 'had':0, 'him':0, 'only':0,
! 455: 'and':0, 'not':0, 'been':0, 'other':0, 'even':0, 'are':0, 'was':0,
! 456: 'new':0, 'most':0, 'but':0, 'when':0, 'some':0, 'made':0, 'from':0,
! 457: 'who':0, 'could':0, 'after':0, 'that':0, 'will':0, 'time':0, 'also':0,
! 458: 'have':0, 'more':0, 'these':0, 'did':0, 'was':0, 'two':0, 'many':0,
! 459: 'they':0, 'may':0, 'before':0, 'for':0, 'which':0, 'out':0, 'then':0,
! 460: 'must':0, 'one':0, 'through':0, 'with':0, 'you':0, 'said':0,
! 461: 'first':0, 'back':0, 'were':0, 'what':0, 'any':0, 'years':0, 'his':0,
! 462: 'her':0, 'where':0, 'all':0, 'its':0, 'now':0, 'much':0, 'she':0,
! 463: 'about':0, 'such':0, 'your':0, 'there':0, 'into':0, 'like':0, 'may':0,
! 464: 'would':0, 'than':0, 'our':0, 'well':0, 'their':0, 'them':0, 'over':0,
! 465: 'down':0,
! 466: 'net':0, 'www':0, 'bad':0, 'Okay':0, 'bin':0, 'cur':0,
! 467: }
! 468:
! 469: wordsDict = {}
! 470: wordsDictHTML = {}
! 471: wordsDictArchive = {}
! 472:
! 473: def cleanupWordsString(str):
! 474: str = string.replace(str, ".", " ")
! 475: str = string.replace(str, "!", " ")
! 476: str = string.replace(str, "?", " ")
! 477: str = string.replace(str, ",", " ")
! 478: str = string.replace(str, "'", " ")
! 479: str = string.replace(str, '"', " ")
! 480: str = string.replace(str, ";", " ")
! 481: str = string.replace(str, "(", " ")
! 482: str = string.replace(str, ")", " ")
! 483: str = string.replace(str, "{", " ")
! 484: str = string.replace(str, "}", " ")
! 485: str = string.replace(str, "<", " ")
! 486: str = string.replace(str, ">", " ")
! 487: str = string.replace(str, "=", " ")
! 488: str = string.replace(str, "/", " ")
! 489: str = string.replace(str, "*", " ")
! 490: str = string.replace(str, ":", " ")
! 491: str = string.replace(str, "#", " ")
! 492: str = string.replace(str, "\\", " ")
! 493: str = string.replace(str, "\n", " ")
! 494: str = string.replace(str, "\r", " ")
! 495: str = string.replace(str, "\xc2", " ")
! 496: str = string.replace(str, "\xa0", " ")
! 497: return str
! 498:
! 499: def cleanupDescrString(str):
! 500: str = string.replace(str, "'", " ")
! 501: str = string.replace(str, "\n", " ")
! 502: str = string.replace(str, "\r", " ")
! 503: str = string.replace(str, "\xc2", " ")
! 504: str = string.replace(str, "\xa0", " ")
! 505: l = string.split(str)
! 506: str = string.join(str)
! 507: return str
! 508:
! 509: def splitIdentifier(str):
! 510: ret = []
! 511: while str != "":
! 512: cur = string.lower(str[0])
! 513: str = str[1:]
! 514: if ((cur < 'a') or (cur > 'z')):
! 515: continue
! 516: while (str != "") and (str[0] >= 'A') and (str[0] <= 'Z'):
! 517: cur = cur + string.lower(str[0])
! 518: str = str[1:]
! 519: while (str != "") and (str[0] >= 'a') and (str[0] <= 'z'):
! 520: cur = cur + str[0]
! 521: str = str[1:]
! 522: while (str != "") and (str[0] >= '0') and (str[0] <= '9'):
! 523: str = str[1:]
! 524: ret.append(cur)
! 525: return ret
! 526:
! 527: def addWord(word, module, symbol, relevance):
! 528: global wordsDict
! 529:
! 530: if word == None or len(word) < 3:
! 531: return -1
! 532: if module == None or symbol == None:
! 533: return -1
! 534: if dropWords.has_key(word):
! 535: return 0
! 536: if ord(word[0]) > 0x80:
! 537: return 0
! 538:
! 539: if wordsDict.has_key(word):
! 540: d = wordsDict[word]
! 541: if d == None:
! 542: return 0
! 543: if len(d) > 500:
! 544: wordsDict[word] = None
! 545: return 0
! 546: try:
! 547: relevance = relevance + d[(module, symbol)]
! 548: except:
! 549: pass
! 550: else:
! 551: wordsDict[word] = {}
! 552: wordsDict[word][(module, symbol)] = relevance
! 553: return relevance
! 554:
! 555: def addString(str, module, symbol, relevance):
! 556: if str == None or len(str) < 3:
! 557: return -1
! 558: ret = 0
! 559: str = cleanupWordsString(str)
! 560: l = string.split(str)
! 561: for word in l:
! 562: if len(word) > 2:
! 563: ret = ret + addWord(word, module, symbol, 5)
! 564:
! 565: return ret
! 566:
! 567: def addWordHTML(word, resource, id, section, relevance):
! 568: global wordsDictHTML
! 569:
! 570: if word == None or len(word) < 3:
! 571: return -1
! 572: if resource == None or section == None:
! 573: return -1
! 574: if dropWords.has_key(word):
! 575: return 0
! 576: if ord(word[0]) > 0x80:
! 577: return 0
! 578:
! 579: section = cleanupDescrString(section)
! 580:
! 581: if wordsDictHTML.has_key(word):
! 582: d = wordsDictHTML[word]
! 583: if d == None:
! 584: print "skipped %s" % (word)
! 585: return 0
! 586: try:
! 587: (r,i,s) = d[resource]
! 588: if i != None:
! 589: id = i
! 590: if s != None:
! 591: section = s
! 592: relevance = relevance + r
! 593: except:
! 594: pass
! 595: else:
! 596: wordsDictHTML[word] = {}
! 597: d = wordsDictHTML[word];
! 598: d[resource] = (relevance, id, section)
! 599: return relevance
! 600:
! 601: def addStringHTML(str, resource, id, section, relevance):
! 602: if str == None or len(str) < 3:
! 603: return -1
! 604: ret = 0
! 605: str = cleanupWordsString(str)
! 606: l = string.split(str)
! 607: for word in l:
! 608: if len(word) > 2:
! 609: try:
! 610: r = addWordHTML(word, resource, id, section, relevance)
! 611: if r < 0:
! 612: print "addWordHTML failed: %s %s" % (word, resource)
! 613: ret = ret + r
! 614: except:
! 615: print "addWordHTML failed: %s %s %d" % (word, resource, relevance)
! 616: print sys.exc_type, sys.exc_value
! 617:
! 618: return ret
! 619:
! 620: def addWordArchive(word, id, relevance):
! 621: global wordsDictArchive
! 622:
! 623: if word == None or len(word) < 3:
! 624: return -1
! 625: if id == None or id == -1:
! 626: return -1
! 627: if dropWords.has_key(word):
! 628: return 0
! 629: if ord(word[0]) > 0x80:
! 630: return 0
! 631:
! 632: if wordsDictArchive.has_key(word):
! 633: d = wordsDictArchive[word]
! 634: if d == None:
! 635: print "skipped %s" % (word)
! 636: return 0
! 637: try:
! 638: r = d[id]
! 639: relevance = relevance + r
! 640: except:
! 641: pass
! 642: else:
! 643: wordsDictArchive[word] = {}
! 644: d = wordsDictArchive[word];
! 645: d[id] = relevance
! 646: return relevance
! 647:
! 648: def addStringArchive(str, id, relevance):
! 649: if str == None or len(str) < 3:
! 650: return -1
! 651: ret = 0
! 652: str = cleanupWordsString(str)
! 653: l = string.split(str)
! 654: for word in l:
! 655: i = len(word)
! 656: if i > 2:
! 657: try:
! 658: r = addWordArchive(word, id, relevance)
! 659: if r < 0:
! 660: print "addWordArchive failed: %s %s" % (word, id)
! 661: else:
! 662: ret = ret + r
! 663: except:
! 664: print "addWordArchive failed: %s %s %d" % (word, id, relevance)
! 665: print sys.exc_type, sys.exc_value
! 666: return ret
! 667:
! 668: #########################################################################
! 669: # #
! 670: # XML API description analysis #
! 671: # #
! 672: #########################################################################
! 673:
! 674: def loadAPI(filename):
! 675: doc = libxml2.parseFile(filename)
! 676: print "loaded %s" % (filename)
! 677: return doc
! 678:
! 679: def foundExport(file, symbol):
! 680: if file == None:
! 681: return 0
! 682: if symbol == None:
! 683: return 0
! 684: addFunction(symbol, file)
! 685: l = splitIdentifier(symbol)
! 686: for word in l:
! 687: addWord(word, file, symbol, 10)
! 688: return 1
! 689:
! 690: def analyzeAPIFile(top):
! 691: count = 0
! 692: name = top.prop("name")
! 693: cur = top.children
! 694: while cur != None:
! 695: if cur.type == 'text':
! 696: cur = cur.next
! 697: continue
! 698: if cur.name == "exports":
! 699: count = count + foundExport(name, cur.prop("symbol"))
! 700: else:
! 701: print "unexpected element %s in API doc <file name='%s'>" % (name)
! 702: cur = cur.next
! 703: return count
! 704:
! 705: def analyzeAPIFiles(top):
! 706: count = 0
! 707: cur = top.children
! 708:
! 709: while cur != None:
! 710: if cur.type == 'text':
! 711: cur = cur.next
! 712: continue
! 713: if cur.name == "file":
! 714: count = count + analyzeAPIFile(cur)
! 715: else:
! 716: print "unexpected element %s in API doc <files>" % (cur.name)
! 717: cur = cur.next
! 718: return count
! 719:
! 720: def analyzeAPIEnum(top):
! 721: file = top.prop("file")
! 722: if file == None:
! 723: return 0
! 724: symbol = top.prop("name")
! 725: if symbol == None:
! 726: return 0
! 727:
! 728: addEnum(symbol, file)
! 729: l = splitIdentifier(symbol)
! 730: for word in l:
! 731: addWord(word, file, symbol, 10)
! 732:
! 733: return 1
! 734:
! 735: def analyzeAPIConst(top):
! 736: file = top.prop("file")
! 737: if file == None:
! 738: return 0
! 739: symbol = top.prop("name")
! 740: if symbol == None:
! 741: return 0
! 742:
! 743: addConst(symbol, file)
! 744: l = splitIdentifier(symbol)
! 745: for word in l:
! 746: addWord(word, file, symbol, 10)
! 747:
! 748: return 1
! 749:
! 750: def analyzeAPIType(top):
! 751: file = top.prop("file")
! 752: if file == None:
! 753: return 0
! 754: symbol = top.prop("name")
! 755: if symbol == None:
! 756: return 0
! 757:
! 758: addType(symbol, file)
! 759: l = splitIdentifier(symbol)
! 760: for word in l:
! 761: addWord(word, file, symbol, 10)
! 762: return 1
! 763:
! 764: def analyzeAPIFunctype(top):
! 765: file = top.prop("file")
! 766: if file == None:
! 767: return 0
! 768: symbol = top.prop("name")
! 769: if symbol == None:
! 770: return 0
! 771:
! 772: addFunctype(symbol, file)
! 773: l = splitIdentifier(symbol)
! 774: for word in l:
! 775: addWord(word, file, symbol, 10)
! 776: return 1
! 777:
! 778: def analyzeAPIStruct(top):
! 779: file = top.prop("file")
! 780: if file == None:
! 781: return 0
! 782: symbol = top.prop("name")
! 783: if symbol == None:
! 784: return 0
! 785:
! 786: addStruct(symbol, file)
! 787: l = splitIdentifier(symbol)
! 788: for word in l:
! 789: addWord(word, file, symbol, 10)
! 790:
! 791: info = top.prop("info")
! 792: if info != None:
! 793: info = string.replace(info, "'", " ")
! 794: info = string.strip(info)
! 795: l = string.split(info)
! 796: for word in l:
! 797: if len(word) > 2:
! 798: addWord(word, file, symbol, 5)
! 799: return 1
! 800:
! 801: def analyzeAPIMacro(top):
! 802: file = top.prop("file")
! 803: if file == None:
! 804: return 0
! 805: symbol = top.prop("name")
! 806: if symbol == None:
! 807: return 0
! 808: symbol = string.replace(symbol, "'", " ")
! 809: symbol = string.strip(symbol)
! 810:
! 811: info = None
! 812: cur = top.children
! 813: while cur != None:
! 814: if cur.type == 'text':
! 815: cur = cur.next
! 816: continue
! 817: if cur.name == "info":
! 818: info = cur.content
! 819: break
! 820: cur = cur.next
! 821:
! 822: l = splitIdentifier(symbol)
! 823: for word in l:
! 824: addWord(word, file, symbol, 10)
! 825:
! 826: if info == None:
! 827: addMacro(symbol, file)
! 828: print "Macro %s description has no <info>" % (symbol)
! 829: return 0
! 830:
! 831: info = string.replace(info, "'", " ")
! 832: info = string.strip(info)
! 833: addMacro(symbol, file, info)
! 834: l = string.split(info)
! 835: for word in l:
! 836: if len(word) > 2:
! 837: addWord(word, file, symbol, 5)
! 838: return 1
! 839:
! 840: def analyzeAPIFunction(top):
! 841: file = top.prop("file")
! 842: if file == None:
! 843: return 0
! 844: symbol = top.prop("name")
! 845: if symbol == None:
! 846: return 0
! 847:
! 848: symbol = string.replace(symbol, "'", " ")
! 849: symbol = string.strip(symbol)
! 850: info = None
! 851: cur = top.children
! 852: while cur != None:
! 853: if cur.type == 'text':
! 854: cur = cur.next
! 855: continue
! 856: if cur.name == "info":
! 857: info = cur.content
! 858: elif cur.name == "return":
! 859: rinfo = cur.prop("info")
! 860: if rinfo != None:
! 861: rinfo = string.replace(rinfo, "'", " ")
! 862: rinfo = string.strip(rinfo)
! 863: addString(rinfo, file, symbol, 7)
! 864: elif cur.name == "arg":
! 865: ainfo = cur.prop("info")
! 866: if ainfo != None:
! 867: ainfo = string.replace(ainfo, "'", " ")
! 868: ainfo = string.strip(ainfo)
! 869: addString(ainfo, file, symbol, 5)
! 870: name = cur.prop("name")
! 871: if name != None:
! 872: name = string.replace(name, "'", " ")
! 873: name = string.strip(name)
! 874: addWord(name, file, symbol, 7)
! 875: cur = cur.next
! 876: if info == None:
! 877: print "Function %s description has no <info>" % (symbol)
! 878: addFunction(symbol, file, "")
! 879: else:
! 880: info = string.replace(info, "'", " ")
! 881: info = string.strip(info)
! 882: addFunction(symbol, file, info)
! 883: addString(info, file, symbol, 5)
! 884:
! 885: l = splitIdentifier(symbol)
! 886: for word in l:
! 887: addWord(word, file, symbol, 10)
! 888:
! 889: return 1
! 890:
! 891: def analyzeAPISymbols(top):
! 892: count = 0
! 893: cur = top.children
! 894:
! 895: while cur != None:
! 896: if cur.type == 'text':
! 897: cur = cur.next
! 898: continue
! 899: if cur.name == "macro":
! 900: count = count + analyzeAPIMacro(cur)
! 901: elif cur.name == "function":
! 902: count = count + analyzeAPIFunction(cur)
! 903: elif cur.name == "const":
! 904: count = count + analyzeAPIConst(cur)
! 905: elif cur.name == "typedef":
! 906: count = count + analyzeAPIType(cur)
! 907: elif cur.name == "struct":
! 908: count = count + analyzeAPIStruct(cur)
! 909: elif cur.name == "enum":
! 910: count = count + analyzeAPIEnum(cur)
! 911: elif cur.name == "functype":
! 912: count = count + analyzeAPIFunctype(cur)
! 913: else:
! 914: print "unexpected element %s in API doc <files>" % (cur.name)
! 915: cur = cur.next
! 916: return count
! 917:
! 918: def analyzeAPI(doc):
! 919: count = 0
! 920: if doc == None:
! 921: return -1
! 922: root = doc.getRootElement()
! 923: if root.name != "api":
! 924: print "Unexpected root name"
! 925: return -1
! 926: cur = root.children
! 927: while cur != None:
! 928: if cur.type == 'text':
! 929: cur = cur.next
! 930: continue
! 931: if cur.name == "files":
! 932: pass
! 933: # count = count + analyzeAPIFiles(cur)
! 934: elif cur.name == "symbols":
! 935: count = count + analyzeAPISymbols(cur)
! 936: else:
! 937: print "unexpected element %s in API doc" % (cur.name)
! 938: cur = cur.next
! 939: return count
! 940:
! 941: #########################################################################
! 942: # #
! 943: # Web pages parsing and analysis #
! 944: # #
! 945: #########################################################################
! 946:
! 947: import glob
! 948:
! 949: def analyzeHTMLText(doc, resource, p, section, id):
! 950: words = 0
! 951: try:
! 952: content = p.content
! 953: words = words + addStringHTML(content, resource, id, section, 5)
! 954: except:
! 955: return -1
! 956: return words
! 957:
! 958: def analyzeHTMLPara(doc, resource, p, section, id):
! 959: words = 0
! 960: try:
! 961: content = p.content
! 962: words = words + addStringHTML(content, resource, id, section, 5)
! 963: except:
! 964: return -1
! 965: return words
! 966:
! 967: def analyzeHTMLPre(doc, resource, p, section, id):
! 968: words = 0
! 969: try:
! 970: content = p.content
! 971: words = words + addStringHTML(content, resource, id, section, 5)
! 972: except:
! 973: return -1
! 974: return words
! 975:
! 976: def analyzeHTML(doc, resource, p, section, id):
! 977: words = 0
! 978: try:
! 979: content = p.content
! 980: words = words + addStringHTML(content, resource, id, section, 5)
! 981: except:
! 982: return -1
! 983: return words
! 984:
! 985: def analyzeHTML(doc, resource):
! 986: para = 0;
! 987: ctxt = doc.xpathNewContext()
! 988: try:
! 989: res = ctxt.xpathEval("//head/title")
! 990: title = res[0].content
! 991: except:
! 992: title = "Page %s" % (resource)
! 993: addPage(resource, title)
! 994: try:
! 995: items = ctxt.xpathEval("//h1 | //h2 | //h3 | //text()")
! 996: section = title
! 997: id = ""
! 998: for item in items:
! 999: if item.name == 'h1' or item.name == 'h2' or item.name == 'h3':
! 1000: section = item.content
! 1001: if item.prop("id"):
! 1002: id = item.prop("id")
! 1003: elif item.prop("name"):
! 1004: id = item.prop("name")
! 1005: elif item.type == 'text':
! 1006: analyzeHTMLText(doc, resource, item, section, id)
! 1007: para = para + 1
! 1008: elif item.name == 'p':
! 1009: analyzeHTMLPara(doc, resource, item, section, id)
! 1010: para = para + 1
! 1011: elif item.name == 'pre':
! 1012: analyzeHTMLPre(doc, resource, item, section, id)
! 1013: para = para + 1
! 1014: else:
! 1015: print "Page %s, unexpected %s element" % (resource, item.name)
! 1016: except:
! 1017: print "Page %s: problem analyzing" % (resource)
! 1018: print sys.exc_type, sys.exc_value
! 1019:
! 1020: return para
! 1021:
! 1022: def analyzeHTMLPages():
! 1023: ret = 0
! 1024: HTMLfiles = glob.glob("*.html") + glob.glob("tutorial/*.html")
! 1025: for html in HTMLfiles:
! 1026: if html[0:3] == "API":
! 1027: continue
! 1028: if html == "xml.html":
! 1029: continue
! 1030: try:
! 1031: doc = libxml2.parseFile(html)
! 1032: except:
! 1033: doc = libxml2.htmlParseFile(html, None)
! 1034: try:
! 1035: res = analyzeHTML(doc, html)
! 1036: print "Parsed %s : %d paragraphs" % (html, res)
! 1037: ret = ret + 1
! 1038: except:
! 1039: print "could not parse %s" % (html)
! 1040: return ret
! 1041:
! 1042: #########################################################################
! 1043: # #
! 1044: # Mail archives parsing and analysis #
! 1045: # #
! 1046: #########################################################################
! 1047:
! 1048: import time
! 1049:
! 1050: def getXMLDateArchive(t = None):
! 1051: if t == None:
! 1052: t = time.time()
! 1053: T = time.gmtime(t)
! 1054: month = time.strftime("%B", T)
! 1055: year = T[0]
! 1056: url = "http://mail.gnome.org/archives/xml/%d-%s/date.html" % (year, month)
! 1057: return url
! 1058:
! 1059: def scanXMLMsgArchive(url, title, force = 0):
! 1060: if url == None or title == None:
! 1061: return 0
! 1062:
! 1063: ID = checkXMLMsgArchive(url)
! 1064: if force == 0 and ID != -1:
! 1065: return 0
! 1066:
! 1067: if ID == -1:
! 1068: ID = addXMLMsgArchive(url, title)
! 1069: if ID == -1:
! 1070: return 0
! 1071:
! 1072: try:
! 1073: print "Loading %s" % (url)
! 1074: doc = libxml2.htmlParseFile(url, None);
! 1075: except:
! 1076: doc = None
! 1077: if doc == None:
! 1078: print "Failed to parse %s" % (url)
! 1079: return 0
! 1080:
! 1081: addStringArchive(title, ID, 20)
! 1082: ctxt = doc.xpathNewContext()
! 1083: texts = ctxt.xpathEval("//pre//text()")
! 1084: for text in texts:
! 1085: addStringArchive(text.content, ID, 5)
! 1086:
! 1087: return 1
! 1088:
! 1089: def scanXMLDateArchive(t = None, force = 0):
! 1090: global wordsDictArchive
! 1091:
! 1092: wordsDictArchive = {}
! 1093:
! 1094: url = getXMLDateArchive(t)
! 1095: print "loading %s" % (url)
! 1096: try:
! 1097: doc = libxml2.htmlParseFile(url, None);
! 1098: except:
! 1099: doc = None
! 1100: if doc == None:
! 1101: print "Failed to parse %s" % (url)
! 1102: return -1
! 1103: ctxt = doc.xpathNewContext()
! 1104: anchors = ctxt.xpathEval("//a[@href]")
! 1105: links = 0
! 1106: newmsg = 0
! 1107: for anchor in anchors:
! 1108: href = anchor.prop("href")
! 1109: if href == None or href[0:3] != "msg":
! 1110: continue
! 1111: try:
! 1112: links = links + 1
! 1113:
! 1114: msg = libxml2.buildURI(href, url)
! 1115: title = anchor.content
! 1116: if title != None and title[0:4] == 'Re: ':
! 1117: title = title[4:]
! 1118: if title != None and title[0:6] == '[xml] ':
! 1119: title = title[6:]
! 1120: newmsg = newmsg + scanXMLMsgArchive(msg, title, force)
! 1121:
! 1122: except:
! 1123: pass
! 1124:
! 1125: return newmsg
! 1126:
! 1127:
! 1128: #########################################################################
! 1129: # #
! 1130: # Main code: open the DB, the API XML and analyze it #
! 1131: # #
! 1132: #########################################################################
! 1133: def analyzeArchives(t = None, force = 0):
! 1134: global wordsDictArchive
! 1135:
! 1136: ret = scanXMLDateArchive(t, force)
! 1137: print "Indexed %d words in %d archive pages" % (len(wordsDictArchive), ret)
! 1138:
! 1139: i = 0
! 1140: skipped = 0
! 1141: for word in wordsDictArchive.keys():
! 1142: refs = wordsDictArchive[word]
! 1143: if refs == None:
! 1144: skipped = skipped + 1
! 1145: continue;
! 1146: for id in refs.keys():
! 1147: relevance = refs[id]
! 1148: updateWordArchive(word, id, relevance)
! 1149: i = i + 1
! 1150:
! 1151: print "Found %d associations in HTML pages" % (i)
! 1152:
! 1153: def analyzeHTMLTop():
! 1154: global wordsDictHTML
! 1155:
! 1156: ret = analyzeHTMLPages()
! 1157: print "Indexed %d words in %d HTML pages" % (len(wordsDictHTML), ret)
! 1158:
! 1159: i = 0
! 1160: skipped = 0
! 1161: for word in wordsDictHTML.keys():
! 1162: refs = wordsDictHTML[word]
! 1163: if refs == None:
! 1164: skipped = skipped + 1
! 1165: continue;
! 1166: for resource in refs.keys():
! 1167: (relevance, id, section) = refs[resource]
! 1168: updateWordHTML(word, resource, section, id, relevance)
! 1169: i = i + 1
! 1170:
! 1171: print "Found %d associations in HTML pages" % (i)
! 1172:
! 1173: def analyzeAPITop():
! 1174: global wordsDict
! 1175: global API
! 1176:
! 1177: try:
! 1178: doc = loadAPI(API)
! 1179: ret = analyzeAPI(doc)
! 1180: print "Analyzed %d blocs" % (ret)
! 1181: doc.freeDoc()
! 1182: except:
! 1183: print "Failed to parse and analyze %s" % (API)
! 1184: print sys.exc_type, sys.exc_value
! 1185: sys.exit(1)
! 1186:
! 1187: print "Indexed %d words" % (len(wordsDict))
! 1188: i = 0
! 1189: skipped = 0
! 1190: for word in wordsDict.keys():
! 1191: refs = wordsDict[word]
! 1192: if refs == None:
! 1193: skipped = skipped + 1
! 1194: continue;
! 1195: for (module, symbol) in refs.keys():
! 1196: updateWord(word, symbol, refs[(module, symbol)])
! 1197: i = i + 1
! 1198:
! 1199: print "Found %d associations, skipped %d words" % (i, skipped)
! 1200:
! 1201: def usage():
! 1202: print "Usage index.py [--force] [--archive] [--archive-year year] [--archive-month month] [--API] [--docs]"
! 1203: sys.exit(1)
! 1204:
! 1205: def main():
! 1206: try:
! 1207: openMySQL()
! 1208: except:
! 1209: print "Failed to open the database"
! 1210: print sys.exc_type, sys.exc_value
! 1211: sys.exit(1)
! 1212:
! 1213: args = sys.argv[1:]
! 1214: force = 0
! 1215: if args:
! 1216: i = 0
! 1217: while i < len(args):
! 1218: if args[i] == '--force':
! 1219: force = 1
! 1220: elif args[i] == '--archive':
! 1221: analyzeArchives(None, force)
! 1222: elif args[i] == '--archive-year':
! 1223: i = i + 1;
! 1224: year = args[i]
! 1225: months = ["January" , "February", "March", "April", "May",
! 1226: "June", "July", "August", "September", "October",
! 1227: "November", "December"];
! 1228: for month in months:
! 1229: try:
! 1230: str = "%s-%s" % (year, month)
! 1231: T = time.strptime(str, "%Y-%B")
! 1232: t = time.mktime(T) + 3600 * 24 * 10;
! 1233: analyzeArchives(t, force)
! 1234: except:
! 1235: print "Failed to index month archive:"
! 1236: print sys.exc_type, sys.exc_value
! 1237: elif args[i] == '--archive-month':
! 1238: i = i + 1;
! 1239: month = args[i]
! 1240: try:
! 1241: T = time.strptime(month, "%Y-%B")
! 1242: t = time.mktime(T) + 3600 * 24 * 10;
! 1243: analyzeArchives(t, force)
! 1244: except:
! 1245: print "Failed to index month archive:"
! 1246: print sys.exc_type, sys.exc_value
! 1247: elif args[i] == '--API':
! 1248: analyzeAPITop()
! 1249: elif args[i] == '--docs':
! 1250: analyzeHTMLTop()
! 1251: else:
! 1252: usage()
! 1253: i = i + 1
! 1254: else:
! 1255: usage()
! 1256:
! 1257: if __name__ == "__main__":
! 1258: main()
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to index.py CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / libxml2 / doc