qm-search-v0.5/0000755000077000007700000000000010405361412013312 5ustar jfentonjfentonqm-search-v0.5/build/0000755000077000007700000000000010405361750014416 5ustar jfentonjfentonqm-search-v0.5/harvesters/0000755000077000007700000000000010405357765015520 5ustar jfentonjfentonqm-search-v0.5/harvesters/harvest.pl0000755000077000007700000001333310404333133017515 0ustar jfentonjfenton#!/usr/bin/perl #for amsouth data use http://www.americansouth.org/oai/index.php use lib 'OAI-Harvester-0.99/lib'; use Net::OAI::Harvester; use XML::LibXML; use Switch; use Utility; $slashchar = '-'; # replace '/' chars in identifiers with this. ########################################################################## # harvest.pl # Harvest metadata from the base URL using NET::OAI::HArvester and save # it in data.txt. # dependency on Utlity.pm and Net:OAI harvester. Both should be in the # current directory # Author: # Urvashi Gadi (ugadi@emory.edu) # Date: # 12/14/2005 ########################################################################### # parse a cached response file containing many records # return possibly many metadata fields as a single space-separated string sub getMDValues { my $metadata = shift; my $key = shift; return join(' ',@{$metadata->{$key}}); } sub formatDate { $date_old = $_[0]; #20051213 $pattern = '\d\d\d\d(\d).(?)(\d).(?)'; #2005-12-13 $pattern1 = '\d\d\d\d-(\d).(?)-(\d).(?)'; #2005/12/13 $pattern2= '\d\d\d\d/(\d).(?)/(\d).(?)'; if ($date_old =~ m/$pattern/ ) { #do nothing } elsif ($date_old =~ m/$pattern1/ ) { @pieces = split(/-/, $date_old); $date_formatted = ""; foreach $piece(@pieces) { $date_formatted = $date_formatted.$piece; } return $date_formatted; } elsif ($date_old =~ m/$pattern2/ ) { @pieces = split(/\//, $date_old); $date_formatted = ""; foreach $piece(@pieces) { $date_formatted = $date_formatted.$piece; } return $date_formatted; } else {print "Alert! date format not take care of!!!";} } sub main { my $filename = "data.txt"; my $baseURL = $_[0]; if(open fileHandle,">$filename") { print "data.txt opened for writing\n"; chmod (0666,$filename); } else {print "data.txt could not be opened. Please chk permissions"}; #Utility constructor my $util = OAI::Utility->new(); my $harvester = Net::OAI::Harvester->new( baseURL => $baseURL ); # loop over sets my $sets = $harvester->listSets(); foreach my $spec ($sets->setSpecs() ) { print "set spec: $spec ; set name: ", $sets->setName( $spec ), "\n"; my $list = $harvester->listAllRecords( metadataPrefix => 'oai_dc', set => $spec); while (my $record = $list->next()) { my $header = $record->header(); my $metadata = $record->metadata(); my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime(time); my $datestamp = sprintf('%d-%02d-%02dT%02d:%02d:%02dZ', $year+1900, $mon+1, $mday, $hour, $min, $sec); print fileHandle "datestamp= ".$datestamp."\n"; print fileHandle "identifier= ".$header->identifier()."\n"; if ($util->lclean(getMDValues($metadata, 'identifier')) ne "") { my $dc_identifier = $util->lclean(getMDValues($metadata, 'identifier')); if (substr($dc_identifier, 0,7) eq "http://") { print fileHandle "url=".$dc_identifier."\n"; } else { print fileHandle "dc_identifier=".$dc_identifier."\n"; } } if ($util->lclean(getMDValues($metadata, 'title')) ne "") { print fileHandle "title=".$util->lclean(getMDValues($metadata, 'title'))."\n"; } if ($util->lclean(getMDValues($metadata, 'description')) ne "") {print fileHandle "description=".$util->lclean(getMDValues($metadata, 'description'))."\n";} if ($util->lclean(getMDValues($metadata, 'subject')) ne "") {print fileHandle "subject=".$util->lclean(getMDValues($metadata, 'subject'))."\n";} if ($util->lclean(getMDValues($metadata, 'creator')) ne "") {print fileHandle "creator=".$util->lclean(getMDValues($metadata, 'creator'))."\n";} if ($util->lclean(getMDValues($metadata, 'contributor')) ne "") {print fileHandle "contributor=".$util->lclean(getMDValues($metadata, 'contributor'))."\n";} if ($util->lclean(getMDValues($metadata, 'publisher')) ne "") {print fileHandle "publisher=".$util->lclean(getMDValues($metadata, 'publisher'))."\n";} if ($util->lclean(getMDValues($metadata, 'date')) ne "") { $date = formatDate(getMDValues($metadata, 'date')); {print fileHandle "date=".$date."\n";} } if ($util->lclean(getMDValues($metadata, 'type')) ne "") {print fileHandle "type=".$util->lclean(getMDValues($metadata, 'type'))."\n";} if ($util->lclean(getMDValues($metadata, 'format')) ne "") {print fileHandle "format=".$util->lclean(getMDValues($metadata, 'format'))."\n";} if ($util->lclean(getMDValues($metadata, 'source')) ne "") {print fileHandle "source=".$util->lclean(getMDValues($metadata, 'source'))."\n";} if ($util->lclean(getMDValues($metadata, 'language')) ne "") {print fileHandle "language=".$util->lclean(getMDValues($metadata, 'language'))."\n";} if ($util->lclean(getMDValues($metadata, 'relation')) ne "") {print fileHandle "relation=".$util->lclean(getMDValues($metadata, 'relation'))."\n";} if ($util->lclean(getMDValues($metadata, 'coverage')) ne "") {print fileHandle "coverage=".$util->lclean(getMDValues($metadata, 'coverage'))."\n";} if ($util->lclean(getMDValues($metadata, 'rights')) ne "") {print fileHandle "rights=".$util->lclean(getMDValues($metadata, 'rights'))."\n";} print fileHandle "oaiset=".($sets->setName( $spec ))."\n"; print fileHandle "granularity=record\n"; print fileHandle "kind=record\n"; print fileHandle "\n"; } } #close file handle close fileHandle; } my $numArgs = $#ARGV + 1; my $baseURL = $ARGV[0]; main($baseURL); qm-search-v0.5/harvesters/get_amsouth_collection_md.pl0000755000077000007700000000711410405357765023275 0ustar jfentonjfenton#!/usr/bin/perl use XML::LibXML; use Switch; ############################################################################## # # Get Amsouth Collection Metadata # ------------------------------- # Aaron Krowne # 2006-02-01 # The Quality Metrics Project # ------------------------------- # # This script grabs the necessary fields from the "archive" table of the live # Amsouth PKP instance. It then outputs Dublin Core renderings of the # collections therein, as separate XML files. # ############################################################################## use DBI; sub formatDate { $date_old = $_[0]; #20051213 $pattern = '\d\d\d\d(\d).(?)(\d).(?)'; #2005-12-13 $pattern1 = '\d\d\d\d-(\d).(?)-(\d).(?)'; #2005/12/13 $pattern2= '\d\d\d\d/(\d).(?)/(\d).(?)'; if ($date_old =~ m/$pattern/ ) { #do nothing } elsif ($date_old =~ m/$pattern1/ ) { @pieces = split(/-/, $date_old); $date_formatted = ""; foreach $piece(@pieces) { $date_formatted = $date_formatted.$piece; } return $date_formatted; } elsif ($date_old =~ m/$pattern2/ ) { @pieces = split(/\//, $date_old); $date_formatted = ""; foreach $piece(@pieces) { $date_formatted = $date_formatted.$piece; } return $date_formatted; } else {print "Alert! date format not take care of!!!";} } sub main { my $output_dir = "amsouth_collection_records"; my $dbh = DBI->connect("DBI:mysql:database=harvester;hostname=callimachus.library.emory.edu","username",""password) or die "couldn't connect to DB!"; if (! -e $output_dir) { mkdir($output_dir); } my $sth = $dbh->prepare("select id, name, repositoryname, url, oai_set, description, rst as subject from archives"); $sth->execute(); my $filename = "data1.txt"; if(open XMLREC,">$filename") { print "data.txt opened for writing\n"; chmod (0666,$filename); } else {print "data.txt could not be opened. Please chk permissions"}; while (my $row = $sth->fetchrow_hashref()) { # TODO: query up archive to retrieve identifiers that are in the archive, # then do "relation" tags for them. # times & dates # my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime(time); my $datestamp = sprintf('%d-%02d-%02dT%02d:%02d:%02dZ', $year+1900, $mon+1, $mday, $hour, $min, $sec); my $date = sprintf('%d-%02d-%02d', $year+1900, $mon+1, $mday); # format for other metadata fields # my $oaiid = "oai:AmSouth:Collection_$row->{id}"; my $desc = $row->{'description'}; $desc =~ s/^\s*//; $desc =~ s/\s*$//; $desc =~ s/\s+/ /g; # output the metadata # print XMLREC "identifier=$oaiid\n"; if (substr($row->{url}, 0, 7) eq "http://") { print XMLREC "url=$row->{url}\n"; } else { print XMLREC "dc_identifier=$row->{url}\n"; } print XMLREC "datestamp=$datestamp\n"; print XMLREC "title=$row->{name}\n"; print XMLREC "subject=$row->{subject}\n"; print XMLREC "description=$desc\n"; $date = formatDate($date); print XMLREC "date=$date\n"; print XMLREC "publisher=somebody\n"; print XMLREC "format=text/html\n"; print XMLREC "language=en-us\n"; print XMLREC "granularity=collection\n"; print XMLREC "kind=collection\n"; print XMLREC "\n"; } close fileHandle; $sth->finish(); $dbh->disconnect(); } main(); qm-search-v0.5/harvesters/sample_data.txt0000644000077000007700000001564410405357172020535 0ustar jfentonjfentondatestamp= 2006-03-06T19:41:50Z identifier=oai:harvester.americansouth.org:record/1 url=http://www.americansouth.org/viewrecord.php?id=1 title=Educational toys in America, 1800 to the present / Karen Hewitt and Louise Roomet. description=Catalogue of a traveling exhibition held at the Robert Hull Fleming Museum, the Sterling and Francine Clark Institute of Art, and the Portland Museum of Art, March 12, 1979 to October 14, 1979. Includes bibliographical references. subject=Educational toys United States Exhibitions. creator=Hewitt, Karen.; ; Roomet, Louise.; ; Robert Hull Fleming Museum.; ; Sterling and Francine Clark Art Institute.; ; Portland Museum of Art.; ; publisher=Burlington, Vt. : Robert Hull Fleming Museum, University of Vermont, c1979. 141 p. : ill. ; 24 cm. date=20030328 type=am source=http://www.atlantahistorycenter.com/ oaiset=Atlantas granularity=record datestamp= 2006-03-06T19:41:50Z identifier=oai:harvester.americansouth.org:record/2 url=http://www.americansouth.org/viewrecord.php?id=2 title=The Railroad in the American landscape, 1850-1950 : the Wellesley College Museum, Wellesley, Massachusetts, 15 April-8 June 1981 / Susan Danly Walther, guest curator. description=Bibliography: p. 143-144. subject=Art, American 19th century Exhibitions. Art, American 20th century Exhibitions.; Railroads in art Exhibitions. creator=Walther, Susan Danly.; ; Wellesley College. Museum.; ; publisher=Wellesley, Mass. : The Museum, [c1981] 144 p. : ill. (some col.) ; 18 x 27 cm. date=20030328 type=am source=http://www.atlantahistorycenter.com/ oaiset=Atlanta History Center - Kenan Research Center granularity=record datestamp=2006-03-06T19:41:50Z identifier=oai:harvester.americansouth.org:record/3 url=http://www.americansouth.org/viewrecord.php?id=3 title=Thomas Jefferson as an architect and a designer of landscapes, by William Alexander Lambeth, M.D., and Warren H. Manning. description=1982.150.1033 subject=Jefferson, Thomas, 1743-1826. Monticello (Va.); University of Virginia.; University of Virginia Buildings. creator=Lambeth, W. A. (William Alexander), b. 1867.; ; Manning, Warren H. (Warren Henry), 1860- joint author.; ; publisher=Boston, New York, Houghton Mifflin Company, 1913. ix, [1], 121, [1] p. incl. plates, port. XXIII pl. (incl. plans, facsims.) 29 cm. date=19961023 type=am source=http://www.atlantahistorycenter.com/ oaiset=Atlanta History Center - Kenan Research Center granularity=record datestamp= 2006-03-06T19:41:50Z identifier= oai:harvester.americansouth.org:record/4 url=http://www.americansouth.org/viewrecord.php?id=4 title=Lesser known architecture of Spain ... with a foreword by F.R. Yerbury. description=V.1:1982.150.970 V.2:1982.150.971 subject=Architecture Spain. creator=Yerbury, Francis Rowland, 1885-1970.; ; publisher=London, E. Benn Ltd., 1925-26. 2 v. 96 pl. 34 cm. date=19961023 type=am source=http://www.atlantahistorycenter.com/ oaiset=Atlanta History Center - Kenan Research Center granularity=record datestamp= 2006-03-06T19:41:50Z identifier= oai:harvester.americansouth.org:record/5 url=http://www.americansouth.org/viewrecord.php?id=5 title=The minor ecclesiastical, domestic, and garden architecture of southern Spain; photographs and drawings by Austin Whittlesey, with a preface by Bertram Grosvenor Goodhue. description=1982.150.284 subject=Architecture Spain. creator=Whittlesey, Austin.; ; publisher=New York, Architectural book Publishing Co., P. Wenzel and M. Krakow [c1917] xi p. 107 pl. front. 32 cm. date=19961023 type=am source=http://www.atlantahistorycenter.com/ oaiset=Atlanta History Center - Kenan Research Center granularity=record datestamp= 2006-03-06T19:41:50Z identifier= oai:harvester.americansouth.org:record/6 url=http://www.americansouth.org/viewrecord.php?id=6 title=The renaissance architectvre of central and northern Spain, a collection of photographs and measvred drawings, by Avstin Whittlesey. description="A companion and continuation of" the author's "The minor ecclesiastical, domestic, and garden architectvre of southern Spain" [c1917] cf. Introd. 1982.150.285 subject=Architecture Spain. creator=Whittlesey, Austin.; ; publisher=New York, Architectvral Book Pvblishing Co., P. Wenzel and M. Krakow, 1920. 2 p. &#x00A0;., ix-xii p. front., 121 pl. 32 cm. date=19961023 type=am source=http://www.atlantahistorycenter.com/ oaiset=Atlanta History Center - Kenan Research Center granularity=record datestamp= 2006-03-06T19:41:50Z identifier= oai:harvester.americansouth.org:record/7 url=http://www.americansouth.org/viewrecord.php?id=7 title=Spanish farm hovses and minor pvblic bvildings; photographs and drawings, by Winsor Sovle...with an introduction by Ralph Adams Cram... description=1982.150.278 subject=Architecture Spain. creator=Soule, Winsor.; ; publisher=New York, Architectvral book pvblishing co., P. Wenzel and M. Krakow, 1924, c1923. 2 p. l., iv sumb. l., 1 l., 98 pl 29 cm. date=19961023 type=am source=http://www.atlantahistorycenter.com/ oaiset=Atlanta History Center - Kenan Research Center granularity=record datestamp= 2006-03-06T19:41:50Z identifier= oai:harvester.americansouth.org:record/8 url=http://www.americansouth.org/viewrecord.php?id=8 title=The Chelsea porcelain toys, scent-bottles, bonbonnieres, etuis, seals and statuettes, made at the Chelsea factory, 1745-1769, & Derby, Chelsea, 1770-1784, described by G.E. Bryant; with 47 plates in colour after water-colour drawings by O.F. Tassart, H. H description=1982.235.780 subject=Chelsea pottery. Derby porcelain.; Chelsea porcelain.; Toys. creator=Bryant, Gilbert Ernest.; ; publisher=London, Boston. The Medici Society, 1925. xii, 193, [8] p. 63 pl. (part col.) 32 x 26 cm. date=19961023 type=am source=http://www.atlantahistorycenter.com/ oaiset=Atlanta History Center - Kenan Research Center granularity=record datestamp= 2006-03-06T19:41:50Z identifier= oai:harvester.americansouth.org:record/9 url=http://www.americansouth.org/viewrecord.php?id=9 title=Bow porcelain, by Frank Hurlbutt ... with sixty-four plates, of which eight are in colour and fifty-six in half-tone. description=Bibliography: p. 159-160. 1982.325.854 subject=Bow porcelain. creator=Hurlbutt, Frank.; ; publisher=London, G. Bell and Sons, ltd., 1926. xviii, 165 p. 64 pl. (part col.; incl. front.) 33 cm. date=19961023 type=am source=http://www.atlantahistorycenter.com/ oaiset=Atlanta History Center - Kenan Research Center granularity=record datestamp= 2006-03-06T19:41:50Z identifier= oai:harvester.americansouth.org:record/10 url=http://www.americansouth.org/viewrecord.php?id=10 title=The old Derby China Factory : the workmen and their productions, containing biographical sketches of the chief artist workmen ... / by John Haslem. description=1982.150.211 subject=Derby porcelain. creator=Haslem, John, 1808-1884.; ; publisher=London : G. Bell, 1876. xiv, 255 p., 11 leaves of col. plates : ill. ; 28 cm. date=19961023 type=am source=http://www.atlantahistorycenter.com/ oaiset=Atlanta History Center - Kenan Research Center granularity=record qm-search-v0.5/src/0000755000077000007700000000000010401420500014067 5ustar jfentonjfentonqm-search-v0.5/src/build.xml0000775000077000007700000000102310401410460015715 0ustar jfentonjfenton qm-search-v0.5/src/index/0000775000077000007700000000000010405365120015212 5ustar jfentonjfentonqm-search-v0.5/src/index/plugins.txt0000644000077000007700000000002710405355456017444 0ustar jfentonjfentonQMAnalysisPlugin_views qm-search-v0.5/src/index/indexDir/0000755000077000007700000000000010405357627016773 5ustar jfentonjfentonqm-search-v0.5/src/index/QMAnalysisPlugin_a.java0000755000077000007700000000265310405355455021576 0ustar jfentonjfenton/*********************************************************************** * QMAnalysisPlugin_a.java * A plug-in implementation for analysis module * * Author: Urvashi Gadi * * Date: 11/1/2005 ***********************************************************************/ import java.util.*; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; public class QMAnalysisPlugin_a implements IQMAnalysisPlugin { public void listAttributes() { } public void addAttribute(Document record, String attributeName) { } public void removeAttribute(Document record, String attributeName) { } public void updateRecordAttribute(Document record, String attributeName) { } public void setRecord(Document record) { } public void getRecord() { } public void updateAllRecords() { } public Document updateRecord(Document record) { //NumberUtils numUtil = new NumberUtils(); // replace later with actual methods that calculate indicators Random generator = new Random(); float r = generator.nextFloat(); Field indicator = Field.Keyword("level", String.valueOf(r)); record.add(indicator); r = generator.nextFloat(); //Field indicator1 = Field.Keyword("hits", String.valueOf(numUtil.pad(r))); Field indicator1 = Field.Keyword("hits", String.valueOf(r)); record.add(indicator1); return record; } public void getPluginName() { } public void getPluginDescription() { } } qm-search-v0.5/src/index/QMAnalysisEngine.java0000755000077000007700000000547210405355455021247 0ustar jfentonjfenton/*************************************************************************** * QMAnalysisEngine.java * Primary class responsible for executing analysis engine. * Configuarble and extensible * Used to determine indicators for data records * Authur: * Urvashi Gadi (ugadi@emory.edu) * Date: * 10/21/2005 *****************************************************************************/ import java.util.*; import java.io.*; import org.apache.lucene.document.Document; class QMAnalysisEngine { static String CONFIGFILE = "config.txt"; // List prescoring plugins void listRegisteredPlugins() {} String getConfiguredPlugin() { String pluginName = null; try { FileInputStream configFileStream = new FileInputStream(CONFIGFILE); BufferedReader in = new BufferedReader(new InputStreamReader(configFileStream)); pluginName = in.readLine(); in.close(); } catch (Exception e) { System.err.println("No configured analysis plugin found"); } return pluginName; } // Add records in digital library record list void addRecords() {} // Add a single record to record list ArrayList addRecord(ArrayList recordList, Document recordToBeAdded) { recordList.add(recordToBeAdded); return recordList; } void removeRecord() {} void removeRecords() {} void readAnalysisPlan() {} IQMAnalysisPlugin loadPlugin(String pluginName) { if (pluginName.equals("")) pluginName = getConfiguredPlugin(); try { SimpleClassLoader plugin = new SimpleClassLoader(); Class loadedPluginClass = plugin.loadClass(pluginName); IQMAnalysisPlugin runnablePlugin = (IQMAnalysisPlugin) loadedPluginClass.newInstance(); return runnablePlugin; } catch (Exception E) { System.out.println("Configured plugin not found"); return null; } } Document updateRecord(Document rec, IQMAnalysisPlugin runnablePlugin) { rec = runnablePlugin.updateRecord(rec); return rec; } void updateRecords(IQMAnalysisPlugin runnablePlugin) { runnablePlugin.updateAllRecords(); } //process all records together to add indicators ArrayList processRecords(ArrayList recordList) { String pluginName = getConfiguredPlugin(); Document newRecord = new Document(); //load configured plugin try { SimpleClassLoader plugin = new SimpleClassLoader(); Class loadedPluginClass = plugin.loadClass(pluginName); IQMAnalysisPlugin runnablePlugin = (IQMAnalysisPlugin) loadedPluginClass.newInstance(); for (int i = 0; i < recordList.size(); i++) { newRecord = (Document)recordList.get(i); newRecord = runnablePlugin.updateRecord(newRecord); recordList.set(i, newRecord); } } catch (Exception E) { System.out.println("Configured plugin not found"); } return recordList; } } qm-search-v0.5/src/index/SimpleClassLoader.java0000755000077000007700000000713410664336543021447 0ustar jfentonjfenton /*************************************************************************** * SimpleClassLoader.java - a bare bones class loader. * * Copyright (c) 1996 Chuck McManis, All Rights Reserved. * * Permission to use, copy, modify, and distribute this software * and its documentation for NON-COMMERCIAL purposes and without * fee is hereby granted provided that this copyright notice * appears in all copies. * * CHUCK MCMANIS MAKES NO REPRESENTATIONS OR WARRANTIES ABOUT THE * SUITABILITY OF THE SOFTWARE, EITHER EXPRESS OR IMPLIED, INCLUDING * BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. CHUCK MCMANIS * SHALL NOT BE LIABLE FOR ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT * OF USING, MODIFYING OR DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES. * * Modified by: * Urvashi Gadi (ugadi@emory.edu) * Date: * 10/20/2005 *****************************************************************************/ import java.util.Hashtable; import java.io.FileInputStream; public class SimpleClassLoader extends ClassLoader { private Hashtable classes = new Hashtable(); public SimpleClassLoader() { } /** * function for reading class implementations * reads them from the local file system */ private byte getClassImplFromDataBase(String className)[] { System.out.println(" >>>>>> Fetching the implementation of "+className); byte result[]; try { FileInputStream fi = new FileInputStream("store\\"+className+".impl"); result = new byte[fi.available()]; fi.read(result); return result; } catch (Exception e) { /* * If we caught an exception, either the class wasnt found or it * was unreadable by our process. */ return null; } } /* * This is a version for external clients since they * will always want the class resolved before it is returned * to them. */ public Class loadClass(String className) throws ClassNotFoundException { return (loadClass(className, true)); } /** * This is the version of loadClass which is called * both from loadClass above and from the internal function * FindClassFromClass. */ public synchronized Class loadClass(String className, boolean resolveIt) throws ClassNotFoundException { Class result; byte classData[]; //System.out.println("\nPlugin Loaded : "+className); /* Check our local cache of classes */ result = (Class)classes.get(className); if (result != null) { System.out.println(" >>>>>> returning cached result."); return result; } /* Check with the primordial class loader */ try { result = super.findSystemClass(className); //System.out.println("returning system class (in CLASSPATH)\n"); return result; } catch (ClassNotFoundException e) { System.out.println(" >>>>>> Not a system class."); } /* Try to load it from repository */ classData = getClassImplFromDataBase(className); if (classData == null) { throw new ClassNotFoundException(); } /* Define it (parse the class file) */ result = defineClass(className, classData, 0, classData.length); if (result == null) { throw new ClassFormatError(); } if (resolveIt) { resolveClass(result); } classes.put(className, result); System.out.println("\nReturning newly loaded class."); return result; } } qm-search-v0.5/src/index/ReadDataFromFile.java0000755000077000007700000001015410405355456021163 0ustar jfentonjfenton/***************************************************************** * ReadDataFromFile.java * Helper class to read data from the harvested data file. * Authur: * Urvashi Gadi (ugadi@emory.edu) * Date: * 10/2/2005 ******************************************************************/ import java.io.*; import java.io.IOException; import java.util.*; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; public class ReadDataFromFile { private final static String DATAFILE = "data.txt"; private BufferedReader in; /* public static void main(String[] args) throws Exception{ System.out.println("reading file"); ReadDataFromFile readFile = new ReadDataFromFile(); QMRecord rec; while ((rec= readFile.readRecord()) != null) { readFile.displayRecordInfo(rec); } } */ /**Open input stream from the input file * @param input file name * */ public ReadDataFromFile(String filename) { try { // open input stream FileInputStream fstream = new FileInputStream(filename); in = new BufferedReader(new InputStreamReader(fstream)); } catch (IOException exp) { System.err.println("File input error "+DATAFILE); } } /**Read next record from the input stream * */ public Document readRecord() { String line; boolean isRecordAvailableToSave = false; if (in == null) { System.out.println("pointer null"); return null; } Document newRecord = null; try { while ((line = in.readLine()) != null) { if (line.length() > 0) { if (newRecord == null) { newRecord = new Document(); } isRecordAvailableToSave = true; AddAttribute(newRecord, line); } else { /* If record exists, save the record */ if (newRecord != null) { isRecordAvailableToSave = false; return(newRecord); } } } if (isRecordAvailableToSave) { return(newRecord); } } catch (IOException exp) { System.err.println("File input error "+DATAFILE); } return null; } /**utility methos to display a record on standard output. * */ public void displayRecordInfo(Document rec) { if (rec == null) { return; } Enumeration attributeList = rec.fields(); Field attributeNameValuePair; while (attributeList.hasMoreElements()) { attributeNameValuePair = (Field)(attributeList.nextElement()); String fieldName = attributeNameValuePair.name(); String fieldValue = attributeNameValuePair.stringValue(); System.out.println(fieldName+"="+fieldValue); } System.out.println("---------end of record-----------"); } /**Method to add attribute to a record * @param record * @param attribute information * */ public void AddAttribute(Document newRecord, String line) { String metadata; String value; if ((newRecord == null ) || (line == null)) { return; } int beginIndex = 0; int endIndex = 0; endIndex = line.indexOf("="); if (endIndex != -1) { metadata = line.substring(beginIndex, endIndex); beginIndex = endIndex+1; endIndex = line.length(); value = line.substring(beginIndex, endIndex); Field data = Field.Keyword(metadata, value); newRecord.add(data); } return; } /* Method to clean up resources */ public void Finalize() { try { in.close(); } catch (IOException exp) { System.err.println("File close error in finalize "+DATAFILE); } } } qm-search-v0.5/src/index/runApp0000755000077000007700000000033510405355456016417 0ustar jfentonjfenton#!/bin/bash #run indexing process for qm-search #command : ./runApp #example for metascholar3 : ./runApp $CLASSPATH indexDir ../../harvesters/data2.txt java -cp .:$1 IndexingApp $2 $3 qm-search-v0.5/src/index/QMAnalysisPlugin_views.java0000755000077000007700000000744310405357555022520 0ustar jfentonjfenton/*********************************************************************** * QMAnalysisPlugin_views.java * A plug-in implementation for analysis module * Adds views information to american south records * * Author: Urvashi Gadi * * Date: 03/1/2006 ***********************************************************************/ import java.util.*; import java.io.*; import java.io.IOException; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.search.*; import org.apache.lucene.index.Term; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.PerFieldAnalyzerWrapper; import org.apache.lucene.analysis.KeywordAnalyzer; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexReader; public class QMAnalysisPlugin_views implements IQMAnalysisPlugin { private final static String DATAFILE = "data/amsouth_view_counts-06_jan.txt"; private final static String INDEXDIR = "indexDir"; private BufferedReader in; public void listAttributes() { } public void addAttribute(Document record, String attributeName) { } public void removeAttribute(Document record, String attributeName) { } public void updateRecordAttribute(Document record, String attributeName) { } public void setRecord(Document record) { } public void getRecord() { } public void updateAllRecords() { try { // open input stream FileInputStream fstream = new FileInputStream(DATAFILE); in = new BufferedReader(new InputStreamReader(fstream)); String line; //search Directory fsDir = FSDirectory.getDirectory(INDEXDIR, false); IndexSearcher is = new IndexSearcher(fsDir); Hits hits; PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper( new StandardAnalyzer()); analyzer.addAnalyzer("identifier", new KeywordAnalyzer()); while (((line = in.readLine()) != null)) { if (line.length() > 0) { // \s is for any whitespace character String temp[] = line.split("\\s"); String views = temp[0]; String identifier = "\" ".concat(temp[1].replace(':', '_')).concat("\""); System.out.println("views = "+views); System.out.println("identifier = "+identifier); try { Query query = QueryParser.parse(identifier, "contents", analyzer); //look for the record with identifier hits = is.search(query); if (hits.length() > 0) { //get a copy of the record Document doc = hits.doc(0); //add views information to the record Field indicator_view = Field.Keyword("views", views); doc.add(indicator_view); //open idex dir to remove old record IndexReader reader = IndexReader.open(INDEXDIR); reader.delete(new Term("identifier", identifier)); reader.close(); //add updated record IndexWriter writer = new IndexWriter(INDEXDIR, analyzer, false); IndexingApp.indexRecord(writer, doc); writer.optimize(); writer.close(); } } catch (ParseException e) { System.out.println("Error --" + e.toString()); } } } } catch (IOException exp) { System.err.println("File input error "+DATAFILE+exp); } } public Document updateRecord(Document record) { return record; } public void getPluginName() { } public void getPluginDescription() { } } qm-search-v0.5/src/index/IQMAnalysisPlugin.java0000755000077000007700000000176510405355455021412 0ustar jfentonjfenton/*********************************************************************** * IQMAnalysisPlugin.java * Primary interface class to be implemented by all analysis * plug-in class * * Author: * Urvashi Gadi * * Date: * 11/1/2005 * Note: * It only has the method names that plugins have to implement. No * implementation ***********************************************************************/ import org.apache.lucene.document.Document; public interface IQMAnalysisPlugin { public void listAttributes(); public void addAttribute(Document searchableRecord, String attributeName); public void removeAttribute(Document searchableRecord, String attributeName); public void updateRecordAttribute(Document searchableRecord, String attributeName); public void setRecord(Document searchableRecord); public void getRecord(); public Document updateRecord(Document seachablerecord); public void getPluginName(); public void getPluginDescription(); public void updateAllRecords(); } qm-search-v0.5/src/index/config.txt0000755000077000007700000000004010405365004017214 0ustar jfentonjfentonQMAnalysisPlugin_classification qm-search-v0.5/src/index/IndexingApp.java0000755000077000007700000001537410405355455020310 0ustar jfentonjfenton/***************************************************************** * IndexingApp.java * Primary Indexing class * Reads data from the harvested data file and indexes it. * Uses different indexing criteria for various metadata. * Applies analysis plugins if specified. * Input: * Indexing directory * data file * Output: * Indexed data in index dir * Dependencies: * Lucene libraries, ReadDataToIndex.java, QMRecord.java, * SimpleClassLoader.java, AttributeNameValuePair.java * QMAnalysisEngine.java, QMAnalysisPlugin.java * Authur: * Urvashi Gadi (ugadi@emory.edu) * Date: * 10/2/2005 ******************************************************************/ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.PerFieldAnalyzerWrapper; import org.apache.lucene.analysis.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import java.io.*; import java.io.IOException; import java.util.*; public class IndexingApp { //number of terms to index in a field private final static int MAX_FIELD_LENGTH = 1000; private final static int MAX_PRECISION_INDICATORS = 10000; //variables to tune indexing performance private final static int INDEXING_MERGE_FACTOR = 1000; private final static int INDEXING_MAX_MERGE_DOCS = 9999999; private final static int INDEXING_MIN_MERGE_DOCS = 1000; private final static String PLUGIN_FILE = "plugins.txt"; public static void main(String[] args) throws Exception { if (args.length != 2) { throw new Exception("Usage: " + IndexingApp.class.getName() + " "); } IndexingApp index = new IndexingApp(); boolean applyPlugin = true; String indexDir = args[0]; String dataFile = args[1]; index.indexData(indexDir, dataFile, applyPlugin); } public void indexData(String indexDirName, String dataFile, boolean applyPlugin) throws IOException{ File indexDir = new File(indexDirName); IndexWriter writer; /* this Analyzer can take diff analyzers for diff fields. If anaylzer for that specific field is not specified, default is used (here it is StandardAnalyzer) */ PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper( new StandardAnalyzer()); analyzer.addAnalyzer("identifier", new KeywordAnalyzer()); writer = new IndexWriter(indexDir, analyzer, true); // set the number of terms to index in one field writer.maxFieldLength = MAX_FIELD_LENGTH; QMAnalysisEngine engine = new QMAnalysisEngine(); // read records from data file, 1 record at a time ReadDataFromFile readFile = new ReadDataFromFile(dataFile); Document rec; IQMAnalysisPlugin runnablePlugin = engine.loadPlugin(""); while ((rec= readFile.readRecord()) != null) { // add quality attributes to the records, uses configured plugin // if flag set if ((applyPlugin) && (runnablePlugin != null)) { rec = engine.updateRecord(rec, runnablePlugin); } try { indexRecord(writer, rec); } catch (IOException e) { System.out.println("Error --" + e.toString()); } } writer.optimize(); writer.close(); applyPlugins(); } //reads plugins.txt to apply analysis plugin to already indexed data private void applyPlugins() { QMAnalysisEngine engine; IQMAnalysisPlugin runnablePlugin; String line; try{ // read config file containing plugin names to applied FileInputStream fstream = new FileInputStream(PLUGIN_FILE); BufferedReader in = new BufferedReader(new InputStreamReader(fstream)); while (((line = in.readLine()) != null)) { if (line.length() > 0) { engine = new QMAnalysisEngine(); runnablePlugin = engine.loadPlugin(line); //apply plugin if ((runnablePlugin != null)) { engine.updateRecords(runnablePlugin); } else System.out.println("plugin:"+line+" not found"); } } }catch (IOException exp) { System.err.println("File input error "+exp); } } //provides indexing criteria for different metadata and indexed records accordingly public static void indexRecord(IndexWriter writer, Document rec) throws IOException { Document doc = new Document(); System.out.println("indexing record with identifer"+rec.get("identifier")); //Add fields to Lucene's Document Enumeration attributeList = rec.fields(); Field attributeNameValuePair; while (attributeList.hasMoreElements()) { attributeNameValuePair = (Field)(attributeList.nextElement()); String fieldName = attributeNameValuePair.name(); String fieldValue = attributeNameValuePair.stringValue(); if ( (fieldName.equalsIgnoreCase("title")) || (fieldName.equalsIgnoreCase("creator")) || (fieldName.equalsIgnoreCase("subject")) || (fieldName.equalsIgnoreCase("description")) || (fieldName.equalsIgnoreCase("contributor")) || (fieldName.equalsIgnoreCase("relations"))) { doc.add(new Field(fieldName, fieldValue, Field.Store.YES, Field.Index.TOKENIZED)); doc.add(new Field("contents", fieldValue, Field.Store.YES, Field.Index.TOKENIZED)); } else if (fieldName.equalsIgnoreCase("identifier")) { doc.add(new Field(fieldName, fieldValue, Field.Store.YES, Field.Index.TOKENIZED)); doc.add(new Field("contents", fieldValue, Field.Store.YES, Field.Index.TOKENIZED)); } else if (fieldName.equalsIgnoreCase("rights")) { doc.add(new Field(fieldName, fieldValue, Field.Store.YES, Field.Index.NO)); doc.add(new Field("contents", fieldValue, Field.Store.YES, Field.Index.NO)); } else if (fieldName.equalsIgnoreCase("date")) { doc.add(new Field(fieldName, fieldValue, Field.Store.YES, Field.Index.UN_TOKENIZED)); doc.add(new Field("contents", fieldValue, Field.Store.YES, Field.Index.UN_TOKENIZED)); } else if ((fieldName.equalsIgnoreCase("publisher")) || (fieldName.equalsIgnoreCase("type")) || (fieldName.equalsIgnoreCase("format")) || (fieldName.equalsIgnoreCase("source")) || (fieldName.equalsIgnoreCase("language")) || (fieldName.equalsIgnoreCase("coverage")) || (fieldName.equalsIgnoreCase("url")) || (fieldName.equalsIgnoreCase("oaiset"))) { doc.add(new Field(fieldName, fieldValue, Field.Store.YES, Field.Index.UN_TOKENIZED)); doc.add(new Field("contents", fieldValue, Field.Store.YES, Field.Index.UN_TOKENIZED)); } // all indicators else { doc.add(new Field(fieldName, fieldValue, Field.Store.YES, Field.Index.UN_TOKENIZED)); doc.add(new Field("contents", fieldValue, Field.Store.YES, Field.Index.UN_TOKENIZED)); } } writer.addDocument(doc); } } qm-search-v0.5/src/index/QMAnalysisPlugin_classification.java0000755000077000007700000000534510405365072024346 0ustar jfentonjfenton/*********************************************************************** * QMAnalysisPlugin_category.java * A plug-in implementation for analysis module * adds classification information to american south data set * * Author: Urvashi Gadi * * Date: 11/1/2005 ***********************************************************************/ import java.util.*; import java.io.*; import java.io.IOException; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; public class QMAnalysisPlugin_category implements IQMAnalysisPlugin { private final static String DATAFILE = "data/amsouth-classified-esc-20060306.txt"; private BufferedReader in; public void listAttributes() { } public void addAttribute(Document record, String attributeName) { } public void removeAttribute(Document record, String attributeName) { } public void updateRecordAttribute(Document record, String attributeName) { } public void setRecord(Document record) { } public void getRecord() { } //constructor public QMAnalysisPlugin_category() { try { // open input stream FileInputStream fstream = new FileInputStream(DATAFILE); in = new BufferedReader(new InputStreamReader(fstream)); } catch (IOException exp) { System.err.println("File input error "+DATAFILE); } } public void updateAllRecords() { } public Document updateRecord(Document record) { // replace later with actual methods that calculate indicators Random generator = new Random(); float r = generator.nextFloat(); Field indicator = Field.Keyword("level", String.valueOf(r)); record.add(indicator); r = generator.nextFloat(); //Field indicator1 = Field.Keyword("hits", String.valueOf(numUtil.pad(r))); Field indicator1 = Field.Keyword("hits", String.valueOf(r)); record.add(indicator1); //get category from text file String line; if (in == null) { System.out.println("pointer null "); return null; } try { //read category information from text file line = in.readLine(); if (line != null) { if (line.length() > 0) { String temp[] = line.split(" "); String categoryCode = temp[0]; Field indicator2 = Field.Keyword("category", categoryCode); //add information to the record record.add(indicator2); } } } catch (IOException exp) { System.err.println("File input error "+DATAFILE); } return record; } public void getPluginName() { } public void getPluginDescription() { } } qm-search-v0.5/src/index/data/0000755000077000007700000000000010405360250016120 5ustar jfentonjfentonqm-search-v0.5/src/search/0000775000077000007700000000000010405360314015350 5ustar jfentonjfentonqm-search-v0.5/src/search/FilterData.java0000644000077000007700000000025610403063047020234 0ustar jfentonjfentonpublic class FilterData{ public String metadata; public String method; public String lowerVal; public String upperVal; public boolean isExcluded; } qm-search-v0.5/src/search/ProcessDim.java0000775000077000007700000013536510401131662020301 0ustar jfentonjfenton/******************************************************************* * ProcessDim.java * Class that transforms search results to XML based on Org * Specifications * Input: search result, org spec list * Output: * formatted output (StringBuffer) * Dependencies: * Authur: * Urvashi Gadi (ugadi@emory.edu) * Date: * 10/2/2005 *******************************************************************/ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.search.*; import java.io.*; import java.util.*; public class ProcessDim { private final static int ORGSPECS_DIMNAME_INDEX = 0; private final static int ORGSPECS_KEY_INDEX = 1; private final static int ORGSPECS_BIN_INDEX = 2; private final static String RESULT_PATH = "http://localhost/qm-search-v0.5/src/search/result"; private int HITS_LIMIT_INDEX = 100; /** process dimensions in case the binning type is natural * @param * @returns number of items in the bin * */ public int processNaturalBinningDim(Hits hits, ArrayList dimList, ArrayList binningFilter, String scoreText, StringBuffer result) { int itemCount = 0; //get dim name and write it to the output file String dimName = (String)dimList.get(ORGSPECS_DIMNAME_INDEX); String temp = "\n"); result = result.append(temp); String binningInfo = (String)dimList.get(ORGSPECS_BIN_INDEX); String keyInfo = (String)dimList.get(ORGSPECS_KEY_INDEX); String key = ""; //final level (last dim) if (dimList.size() == 3) { StringBuffer tempResult = new StringBuffer(); itemCount = itemCount + processFinalBinningLevelNatural(hits, dimList, binningFilter, scoreText, result); result = result.append(tempResult); } else { StringBuffer tempResult = new StringBuffer(); //add key to hits if required, dimName is the key name hits = addKeyToHits(hits, keyInfo, dimName); if (keyInfo.startsWith("combination")) key = dimName; else key = keyInfo; // get total bins at this level Hashtable bins = new Hashtable(); bins = getBins(hits, key); //move on the next level of dimension ArrayList tempDimList = new ArrayList(); tempDimList = getUpdatedDimList(dimList); ArrayList filter = new ArrayList(); String scoreTextTemp = ""; int ordCount = 0; for (Enumeration e = bins.keys() ; e.hasMoreElements() ;) { String hashKey = (String)e.nextElement(); //prepare data for the next dim scoreTextTemp = getUpdatedScoreText(scoreText, dimName, hashKey); filter = getUpdatedFilter(key, hashKey, binningFilter); binningInfo = (String)tempDimList.get(ORGSPECS_BIN_INDEX); int count=0; StringBuffer tempBuffer = new StringBuffer(); if (binningInfo.startsWith("trivial")) { tempBuffer = tempBuffer.append(tempResult); StringBuffer partialResult = new StringBuffer(); count = processTrivialBinningDim(hits, tempDimList, filter, scoreTextTemp, partialResult); tempBuffer = tempBuffer.append(partialResult); } else if (binningInfo.startsWith("natural")) { StringBuffer partialResult = new StringBuffer(); count = processNaturalBinningDim(hits, tempDimList, filter, scoreTextTemp, partialResult); tempBuffer = tempBuffer.append(tempResult); tempBuffer = tempBuffer.append(partialResult); } else if (binningInfo.startsWith("fixed")) { StringBuffer partialResult = new StringBuffer(); count = processFixedBinningDim(hits, tempDimList, filter, scoreTextTemp, partialResult); tempBuffer = tempBuffer.append(tempResult); tempBuffer = tempBuffer.append(partialResult); } itemCount = itemCount + count; if (hashKey.equals("null")) result = result.append("\n"); else result = result.append("\n"); result = result.append(tempBuffer); result = result.append("\n"); ordCount++; } } result = result.append("\n"); return itemCount; } /** update score text, displayed with each item * @param * @returns updated string * */ public String getUpdatedScoreText(String scoreText, String dimName, String key) { String scoreTextTemp = ""; scoreTextTemp = scoreTextTemp.concat(scoreText); scoreTextTemp = scoreTextTemp.concat(""+key+"\n"); return scoreTextTemp; } /** update filter list, used to filter items in bins * @param * @returns updated ArrayList * */ public ArrayList getUpdatedFilter(String key, String hashKey, ArrayList binningFilter) { ArrayList bFilter = new ArrayList(); ArrayList filter = new ArrayList(); bFilter.add(key); bFilter.add(hashKey); filter = copy2DArraylist(binningFilter); filter.add(bFilter); return filter; } /** get dim list for the next level * @param dim list * @returns updated dim list * */ public ArrayList getUpdatedDimList(ArrayList dimList) { ArrayList tempDimList = new ArrayList(); tempDimList = copy1DArraylist(dimList); tempDimList.remove(0); tempDimList.remove(0); tempDimList.remove(0); return tempDimList; } /** process dimensions for fixed binning type * @param * @returns * */ public int processFixedBinningDim(Hits hits, ArrayList dimList, ArrayList binningFilter, String scoreText, StringBuffer result) { //get dim name and write it to the output file String dimName = (String)dimList.get(ORGSPECS_DIMNAME_INDEX); String temp = "\n"); result = result.append(temp); int itemCount=0; // extract fixed binning size String binningInfo = (String)dimList.get(ORGSPECS_BIN_INDEX); int fixedBinSize = Integer.parseInt(binningInfo.split(" ")[1]); String keyInfo = (String)dimList.get(ORGSPECS_KEY_INDEX); String key = ""; int ordCount = 0; ArrayList bFilter = new ArrayList(); ArrayList filter = new ArrayList(); String scoreTextTemp = ""; String binCount = (binningInfo.split(" "))[1]; //final level (last dim) if (dimList.size() == 3) { StringBuffer tempResult = new StringBuffer(); itemCount = itemCount + processFinalBinningLevelFixed(hits, dimList, binningFilter, scoreText, Integer.parseInt(binCount), tempResult); result = result.append(tempResult); } else { StringBuffer tempResult = new StringBuffer(); //add key to hits if required, dimName is the key name hits = addKeyToHits(hits, keyInfo, dimName); if (keyInfo.startsWith("combination")) key = dimName; else key = keyInfo; //move on the next level of dimension ArrayList tempDimList = new ArrayList(); tempDimList = getUpdatedDimList(dimList); if (isKeyValueDecimal(key, hits)) { String member; //get fixed bins for this key ArrayList bins = new ArrayList(); bins = getBinsforDecimalKey(hits, key, fixedBinSize); for (int j = 0; j < bins.size(); j++) { member = ""; member = (String)bins.get(j); bFilter = new ArrayList(); // prepare data for the next level String[] memberList = member.split(" "); for (int k=0; k"; result = result.append(temp); result = result.append(tempBuffer); result = result.append("\n"); ordCount++; } } // non-decimal key else { // get total bins at this level Hashtable bins = new Hashtable(); bins = getBins(hits, key, fixedBinSize); ordCount = 0; for (Enumeration e = bins.keys() ; e.hasMoreElements() ;) { String hashKey = (String)e.nextElement(); //prepare data for the next level of dim scoreTextTemp = getUpdatedScoreText(scoreText, dimName, hashKey); filter = getUpdatedFilter(key, hashKey, binningFilter); binningInfo = (String)tempDimList.get(ORGSPECS_BIN_INDEX); int count=0; StringBuffer tempBuffer = new StringBuffer(); if (binningInfo.startsWith("trivial")) { tempBuffer = tempBuffer.append(tempResult); StringBuffer partialResult = new StringBuffer(); count = processTrivialBinningDim(hits, tempDimList, filter, scoreTextTemp, partialResult); tempBuffer = tempBuffer.append(partialResult); } else if (binningInfo.startsWith("natural")) { StringBuffer partialResult = new StringBuffer(); count = processNaturalBinningDim(hits, tempDimList, filter, scoreTextTemp, partialResult); tempBuffer = tempBuffer.append(tempResult); tempBuffer = tempBuffer.append(partialResult); } else if (binningInfo.startsWith("fixed")) { StringBuffer partialResult = new StringBuffer(); count = processFixedBinningDim(hits, tempDimList, filter, scoreTextTemp, partialResult); tempBuffer = tempBuffer.append(tempResult); tempBuffer = tempBuffer.append(partialResult); } itemCount = itemCount + count; if (hashKey.equals("null")) result = result.append("\n"); else result = result.append("\n"); result = result.append(tempBuffer); result = result.append("\n"); ordCount++; } } } result = result.append("\n"); return itemCount; } /** get total bins for the key * @param hits, key * @returns updated Hashtable * */ public Hashtable getBins(Hits hits, String key) { Hashtable bins = new Hashtable(); try{ int length = 0; if (hits.length() > HITS_LIMIT_INDEX) length = HITS_LIMIT_INDEX; else length = hits.length(); for (int i = 0; i < length; i++) { Document doc = hits.doc(i); String fieldValue = ""; if (key.equals("score")) { fieldValue = String.valueOf(hits.score(i)); } else { if (doc.get(key) != null) fieldValue = (doc.get(key).trim()); } if ( fieldValue != null) { if (!fieldValue.equals("")) { bins.put(fieldValue, fieldValue); } else bins.put("null", "null"); } } return bins; } catch (IOException e) { System.out.println("Error --" + e.toString()); bins.clear(); return bins; } } /** chk if the key contains decimal number or not * modify later...might not be correct * @param key * @param hits * @returns boolean * */ public boolean isKeyValueDecimal(String key, Hits hits) { boolean isDecimal = true; // if DC element then non-decimal if ((key.equalsIgnoreCase("title")) || (key.equalsIgnoreCase("creator")) || (key.equalsIgnoreCase("subject")) || (key.equalsIgnoreCase("description")) || (key.equalsIgnoreCase("contributor")) || (key.equalsIgnoreCase("date")) || (key.equalsIgnoreCase("type")) || (key.equalsIgnoreCase("format")) || (key.equalsIgnoreCase("Identifier")) || (key.equalsIgnoreCase("source")) || (key.equalsIgnoreCase("language")) || (key.equalsIgnoreCase("relation")) || (key.equalsIgnoreCase("coverage")) || (key.equalsIgnoreCase("rights")) || (key.equalsIgnoreCase("publisher"))|| (key.equalsIgnoreCase("datestamp"))|| (key.equalsIgnoreCase("url"))|| (key.equalsIgnoreCase("granularity"))) { isDecimal = false; } //if key is score, value is decimal else if((key.equalsIgnoreCase("score"))){ //do nothing } else { try { //extract a key value and chk if its contains a decimal number of not int length = 0; if (hits.length() > HITS_LIMIT_INDEX) length = HITS_LIMIT_INDEX; else length = hits.length(); String keyValue = ""; for (int i = 0; i < length; i++) { Document doc = hits.doc(i); keyValue = doc.get(key); if ((keyValue != "") || (keyValue != null)) break; } isDecimal = hasOnlyDigits(keyValue); } catch (IOException e) { System.out.println("Error --" + e.toString()); } } return isDecimal; } /** find if string contains a decimal number * @param key, hits * @returns float * */ public boolean hasOnlyDigits(String s) { for (int i=0; i < s.length(); i++) { if (s.charAt(i) == '.') continue; if (!(Character.isDigit(s.charAt(i)))) return false; } return true; } /** find min key value for decimal fixed binning kind * @param key, hits * @returns float * */ public float getMinFieldValue(Hits hits, String key) { float fieldValue = 0; float minValue = 1; try { int length = 0; if (hits.length() > HITS_LIMIT_INDEX) length = HITS_LIMIT_INDEX; else length = hits.length(); for (int i = 0; i < length; i++) { Document doc = hits.doc(i); if (key.equals("score")) { fieldValue = hits.score(i); } else { if (doc.get(key) != null) fieldValue = Float.parseFloat((doc.get(key).trim())); } if (fieldValue < minValue) minValue = fieldValue; } } catch (IOException e) { System.out.println("Error --" + e.toString()); } return minValue; } /** find max key value fo decimal fixed binning kind * @param key, hits * @returns float * */ public float getMaxFieldValue(Hits hits, String key) { float fieldValue = 0; float maxValue = 0; try { int length = 0; if (hits.length() > HITS_LIMIT_INDEX) length = HITS_LIMIT_INDEX; else length = hits.length(); for (int i = 0; i < length; i++) { Document doc = hits.doc(i); if (key.equals("score")) { fieldValue = hits.score(i); } else { if (doc.get(key) != null) fieldValue = Float.parseFloat((doc.get(key)).trim()); } if (fieldValue > maxValue) maxValue = fieldValue; } } catch (IOException e) { System.out.println("Error --" + e.toString()); } return maxValue; } /** get total bins for the non-decimal key for fixed binnig * [overloaded for fixed binning] * @param hits, key * @returns * */ public Hashtable getBins(Hits hits, String key, int binCount) { Hashtable bins = new Hashtable(); try { int length = 0; if (hits.length() > HITS_LIMIT_INDEX) length = HITS_LIMIT_INDEX; else length = hits.length(); for (int i = 0; i < length; i++) { Document doc = hits.doc(i); String fieldValue = ""; if (key.equals("score")) { fieldValue = String.valueOf(hits.score(i)); } else { if (doc.get(key) != null) fieldValue = (doc.get(key)).trim(); } if ( fieldValue != null) { if (!fieldValue.equals("")) { bins.put(fieldValue, fieldValue); } else bins.put("null", "null"); } if (bins.size() == (binCount)) break; } } catch (IOException e) { System.out.println("Error --" + e.toString()); bins.clear(); } return bins; } /** get total bins for fixed binnig. Key is a decimal number * @param hits, key * @returns * */ public ArrayList getBinsforDecimalKey(Hits hits, String key, int binCount) { ArrayList bins = new ArrayList(); String member; float max = Math.round(getMaxFieldValue(hits, key)); float min = getMinFieldValue(hits, key); float range = (max-min)/binCount; float lower = min; float upper = min + range; if (binCount==1) upper=max; for (int i=0; i"); result = result.append(temp); //extract other info from the dimensions list String binningInfo = (String)dimList.get(ORGSPECS_BIN_INDEX); String keyInfo = (String)dimList.get(ORGSPECS_KEY_INDEX); //lowest level dim if (dimList.size() == 3) { StringBuffer tempResult = new StringBuffer(); itemCount = itemCount + processFinalBinningLevel(hits, dimList, binningFilter, scoreText, tempResult); result = result.append(tempResult); } else { StringBuffer tempResult = new StringBuffer(); //add key to hits if required hits = addKeyToHits(hits, keyInfo, dimName); String key=""; if (keyInfo.startsWith("combination")) key = dimName; else key = keyInfo; ArrayList tempDimList = new ArrayList(); tempDimList = getUpdatedDimList(dimList); //prepare data for next dim String scoreTextTemp = ""; scoreTextTemp = scoreTextTemp.concat(scoreText); scoreTextTemp = scoreTextTemp.concat("fill_later "+key+" "+dimName+"\n"); binningInfo = (String)tempDimList.get(ORGSPECS_BIN_INDEX); int count=0; StringBuffer tempBuffer = new StringBuffer(); if (binningInfo.startsWith("trivial")) { tempBuffer = tempBuffer.append(tempResult); StringBuffer partialResult = new StringBuffer(); count = processTrivialBinningDim(hits, tempDimList, binningFilter, scoreTextTemp, partialResult); tempBuffer = tempBuffer.append(partialResult); } else if (binningInfo.startsWith("natural")) { StringBuffer partialResult = new StringBuffer(); count = processNaturalBinningDim(hits, tempDimList, binningFilter, scoreTextTemp, partialResult); tempBuffer = tempBuffer.append(tempResult); tempBuffer = tempBuffer.append(partialResult); } else if (binningInfo.startsWith("fixed")) { StringBuffer partialResult = new StringBuffer(); count = processFixedBinningDim(hits, tempDimList, binningFilter, scoreTextTemp, partialResult); tempBuffer = tempBuffer.append(tempResult); tempBuffer = tempBuffer.append(partialResult); } itemCount = itemCount + count; result = result.append("\n"); result = result.append(tempBuffer); result = result.append("\n"); } result = result.append("\n"); return itemCount; } /** process final level(dimensions) ifor trivial binning type * @param * @returns * */ public int processFinalBinningLevel(Hits hits, ArrayList dimList, ArrayList binningFilter, String scoreText, StringBuffer result) { String dimName = (String)dimList.get(ORGSPECS_DIMNAME_INDEX); String keyInfo = (String)dimList.get(ORGSPECS_KEY_INDEX); int itemCount = 0; String key; if (keyInfo.startsWith("combination")) key = dimName; else key = keyInfo; String scoreTextTemp = ""; scoreTextTemp = scoreTextTemp.concat(scoreText); scoreTextTemp = scoreTextTemp.concat("fill_later "+key+" "+dimName+"\n"); //add key to hits if required hits = addKeyToHits(hits, keyInfo, dimName); /* write hits to output file */ StringBuffer tempResult = new StringBuffer(); itemCount = convertHitsToXml(hits, dimName, binningFilter, scoreTextTemp, tempResult); result = result.append(tempResult); return itemCount; } /** process final level(dimensions) in case the binning type is natural * @param * @returns * */ public int processFinalBinningLevelNatural(Hits hits, ArrayList dimList, ArrayList binningFilter, String scoreText, StringBuffer result) { String dimName = (String)dimList.get(ORGSPECS_DIMNAME_INDEX); String keyInfo = (String)dimList.get(ORGSPECS_KEY_INDEX); int itemCount=0; int dimCount=0; String key = ""; if (keyInfo.startsWith("combination")) key = dimName; else key = keyInfo; //add key to hits if required hits = addKeyToHits(hits, keyInfo, dimName); Hashtable bins = new Hashtable(); bins = getBins(hits, key); ArrayList filter = new ArrayList(); int count = 0; String scoreTextTemp = ""; try { for (Enumeration e = bins.keys() ; e.hasMoreElements() ;) { String hashKey = (String)e.nextElement(); filter = getUpdatedFilter(key, hashKey, binningFilter); scoreTextTemp = getUpdatedScoreText(scoreText, dimName, hashKey); StringBuffer tempResult = new StringBuffer(); itemCount = itemCount + convertHitsToXml(hits, dimName, filter, hashKey, count, scoreTextTemp, tempResult); result = result.append(tempResult); count++; } //chk if any record part of this dim but not any bin boolean inDimFlag = false; StringBuffer tempResult = new StringBuffer(); int length = 0; if (hits.length() > HITS_LIMIT_INDEX) length = HITS_LIMIT_INDEX; else length = hits.length(); for (int i = 0; i < length; i++) { Document doc = hits.doc(i); if (recordOnlyInDim(doc, filter, bins, key, String.valueOf(hits.score(i)))) { dimCount++; inDimFlag = true; StringBuffer partialResult = new StringBuffer(); tempResult = tempResult.append(writeRecordToOutput(doc, dimName, scoreText, key, String.valueOf(hits.score(i)), partialResult)); } } if (inDimFlag) { String temp = "\n"; result = result.append(temp); result = result.append(tempResult); result = result.append("\n"); } } catch (IOException e) { System.out.println("Error --" + e.toString()); } return (itemCount+dimCount); } /** process final level(dimensions) for fixed binning type * @param * @returns * */ public int processFinalBinningLevelFixed(Hits hits, ArrayList dimList, ArrayList binningFilter, String scoreText, int binCount, StringBuffer result) { String dimName = (String)dimList.get(ORGSPECS_DIMNAME_INDEX); String keyInfo = (String)dimList.get(ORGSPECS_KEY_INDEX); String key = ""; int itemCount=0; int dimCount=0; if (keyInfo.startsWith("combination")) key = dimName; else key = keyInfo; //add key to hits if required hits = addKeyToHits(hits, keyInfo, dimName); try { if (isKeyValueDecimal(key, hits)) { ArrayList bins = new ArrayList(); bins = getBinsforDecimalKey(hits, key, binCount); ArrayList bFilter; ArrayList filter = new ArrayList(); String member; String scoreTextTemp = ""; for (int j = 0; j < bins.size(); j++) { member = ""; member = (String)bins.get(j); bFilter = new ArrayList(); String[] memberList = member.split(" "); for (int k=0; k HITS_LIMIT_INDEX) length = HITS_LIMIT_INDEX; else length = hits.length(); for (int i = 0; i < length; i++) { Document doc = hits.doc(i); if (recordOnlyInDim(doc, filter, bins, key, String.valueOf(hits.score(i)))) { dimCount++; inDimFlag = true; StringBuffer partialResult = new StringBuffer(); tempResult = tempResult.append(writeRecordToOutput(doc, dimName, scoreText, key, String.valueOf(hits.score(i)), partialResult)); } } if (inDimFlag) { String temp = "\n"; result = result.append(temp); result = result.append(tempResult); result = result.append("\n"); } } } catch (IOException e) { System.out.println("Error --" + e.toString()); } return (itemCount+dimCount); } // chk if the record belongs to this dim but not in any bin // should satisfy all filters but last public boolean recordOnlyInDim(Document doc, ArrayList binningFilter, Hashtable bins, String key, String scoreValue) { boolean recordInDimFlag = true; ArrayList filter = new ArrayList(); String keyName = ""; String keyValue = ""; for (int i = 0; i < (binningFilter.size()-1); i++) { filter = (ArrayList) binningFilter.get(i); keyValue = (String)filter.get(1); keyName = (String)filter.get(0); String docKeyValue = "null"; if (keyName.equals("score")) { docKeyValue = scoreValue; } else { docKeyValue = doc.get(keyName); } if (docKeyValue != null) { if (!keyValue.equals(docKeyValue.trim())) { recordInDimFlag = false; return recordInDimFlag; } } } for (Enumeration e = bins.keys() ; e.hasMoreElements() ;) { String hashKey = (String)e.nextElement(); keyValue = ""; if (key.equals("score")) keyValue = scoreValue; else keyValue = doc.get(key); if (keyValue != null) { if (hashKey.equals(keyValue.trim())) { recordInDimFlag = false; break; } else recordInDimFlag = true; } } return recordInDimFlag; } /** method to chk if the item is in the bin or not * @param * @returns * */ public boolean recordInBin(Document doc, ArrayList binningFilter, String scoreValue) { boolean recordInBinFlag = true; ArrayList bFilter = new ArrayList(); String key = ""; float lower, upper; String keyValue = ""; for (int i = 0; i < binningFilter.size(); i++) { bFilter = new ArrayList(); bFilter = (ArrayList) binningFilter.get(i); key = (String) bFilter.get(0); if (key.equals("range")) { key = (String)bFilter.get(1); lower = Float.parseFloat((String)bFilter.get(2)); upper = Float.parseFloat((String)bFilter.get(3)); if (key.equals("score")) keyValue = scoreValue; else { if (doc.get(key) != null) keyValue = (doc.get(key)).trim(); } float keyValueDecimal = Float.parseFloat(keyValue); if ((keyValueDecimal >= lower) && (keyValueDecimal < upper)) { } else { recordInBinFlag = false; break; } } else { keyValue = (String)bFilter.get(1); if (key.equals("score")) { if (!keyValue.equals(scoreValue)) { recordInBinFlag = false; break; } } else { String value = "null"; if (key.equals("score")) value = scoreValue; else { if (doc.get(key) != null) { value = (doc.get(key)).trim(); } } if (!keyValue.equals(value)) { recordInBinFlag = false; break; } } } } return recordInBinFlag; } /** method to convert hits to XML for fixed binning with decimal key * @param * @returns * */ public int convertHitsToXmlDecimal(Hits hits, String dimName, ArrayList binningFilter, String keyInfo, int count, String scoreText, StringBuffer result) { String temp = ""; int itemCount = 0; StringBuffer tempResult = new StringBuffer(); try{ String info[] = keyInfo.split(" "); String lower = (String)info[2]; String upper = (String)info[3]; int length = 0; if (hits.length() > HITS_LIMIT_INDEX) length = HITS_LIMIT_INDEX; else length = hits.length(); for (int i = 0; i < length; i++) { Document doc = hits.doc(i); if (recordInBin(doc, binningFilter, String.valueOf(hits.score(i)))) { temp = ""; itemCount++; if (doc.get("identifier") != null) { temp = "\n"); } else temp = "\n"; tempResult = tempResult.append(temp); String[] scoreArray = scoreText.split("\n"); for (int j=0; j"+scoreValue.trim()+"\n"; } else { if (doc.get(key) != null) tempScore = ""+(doc.get(key)).trim()+"\n"; else tempScore = "null\n"; } tempResult = tempResult.append(tempScore); } tempResult = tempResult.append(scoreArray[j]); } tempResult = tempResult.append("\n"); String fieldValue = ""; for (Enumeration docField = doc.fields(); docField.hasMoreElements(); ) { Object a = docField.nextElement(); String data = a.toString(); int pos1 = data.indexOf('<'); int pos2 = data.indexOf(':'); String field = data.substring(pos1+1, pos2); if (!field.equals("contents")) { tempResult = tempResult.append("\n\t<"); tempResult = tempResult.append(field); tempResult = tempResult.append(">"); if (doc.get(field) != null) fieldValue = (doc.get(field)).trim(); if (field.equals("date")) { fieldValue = formatDate(fieldValue); } tempResult = tempResult.append(fieldValue); tempResult = tempResult.append(""); } } tempResult = tempResult.append("\n"); tempResult = tempResult.append("\n"); } } temp = "\n"; result = result.append(temp); result = result.append(tempResult); temp = "\n"; result = result.append(temp); } catch (IOException e) { System.out.println("Error --" + e.toString()); } return itemCount; } /** method to convert hits to XML format for natural binning * @param * @returns item count * */ public int convertHitsToXml(Hits hits, String dimName, ArrayList binningFilter, String keyValue, int count, String scoreText, StringBuffer result) { String temp = ""; int itemCount=0; StringBuffer tempResult = new StringBuffer(); try{ int length = 0; if (hits.length() > HITS_LIMIT_INDEX) length = HITS_LIMIT_INDEX; else length = hits.length(); for (int i = 0; i < length; i++) { Document doc = hits.doc(i); if (recordInBin(doc, binningFilter, String.valueOf(hits.score(i)))) { itemCount++; temp = ""; if (doc.get("identifier") != null) temp = "\n"); else temp = "\n"; tempResult = tempResult.append(temp); String[] scoreArray = scoreText.split("\n"); for (int j=0; j"+scoreValue.trim()+"\n"; } else { if (doc.get(key) != null) tempScore = ""+(doc.get(key)).trim()+"\n"; else tempScore = "null\n"; } tempResult = tempResult.append(tempScore); } else tempResult = tempResult.append(scoreArray[j].getBytes()); } tempResult = tempResult.append("\n"); String fieldValue = ""; for (Enumeration docField = doc.fields(); docField.hasMoreElements(); ) { Object a = docField.nextElement(); String data = a.toString(); int pos1 = data.indexOf('<'); int pos2 = data.indexOf(':'); String field = data.substring(pos1+1, pos2); if (!field.equals("contents")) { tempResult = tempResult.append("\n\t<"); tempResult = tempResult.append(field); tempResult = tempResult.append(">"); if (doc.get(field) != null) fieldValue = (doc.get(field)).trim(); if (field.equals("date")) { fieldValue = formatDate(fieldValue); } tempResult = tempResult.append(fieldValue); tempResult = tempResult.append(""); } } tempResult = tempResult.append("\n"); tempResult = tempResult.append("\n"); } } if (keyValue.equals("null")) temp = "\n"; else temp = "\n"; result=result.append(temp); result=result.append(tempResult); temp = "\n"; result = result.append(temp); } catch (IOException e) { System.out.println("Error --" + e.toString()); } return itemCount; } /** method to convert a single record in XML format * @param * @returns filled StringBuffer * */ public StringBuffer writeRecordToOutput(Document doc, String dimName, String scoreText, String inKey, String scoreValue, StringBuffer result) { String fieldValue; String temp = ""; if (doc.get("identifier") != null) temp = "\n"); else temp = "\n"; result = result.append(temp); if (inKey.equals("score")) { temp = "\n"+scoreValue.trim()+""); } else { if (doc.get(inKey) != null) temp = "\n"+(doc.get(inKey)).trim()+""); else temp = "\nnull"); } result = result.append(temp); String[] scoreArray = scoreText.split("\n"); for (int j=0; j"+scoreValue.trim()+"\n"; } else { if (doc.get(key) != null) tempScore = ""+(doc.get(key)).trim()+"\n"; else tempScore = "null\n"; } result = result.append(tempScore); } else result = result.append(scoreArray[j].getBytes()); } result = result.append("\n"); for (Enumeration docField = doc.fields(); docField.hasMoreElements(); ) { Object a = docField.nextElement(); String data = a.toString(); int pos1 = data.indexOf('<'); int pos2 = data.indexOf(':'); String field = data.substring(pos1+1, pos2); if (!field.equals("contents")) { result = result.append("\n\t<"); result = result.append(field); result = result.append(">"); if (doc.get(field) != null) fieldValue = (doc.get(field)).trim(); else fieldValue = ""; if (field.equals("date")) { fieldValue = formatDate(fieldValue); } result = result.append(fieldValue); result = result.append(""); } } result = result.append("\n"); result = result.append("\n"); return result; } /** method to convert hits to XML format for trivial binning * @param * @returns item count * */ public int convertHitsToXml(Hits hits, String dimName, ArrayList binningFilter, String scoreText, StringBuffer result) { boolean writeRecord = true; String temp = ""; StringBuffer tempResult= new StringBuffer(); int itemCount=0; try{ int length = 0; if (hits.length() > HITS_LIMIT_INDEX) length = HITS_LIMIT_INDEX; else length = hits.length(); for (int i = 0; i < length; i++) { Document doc = hits.doc(i); float score = hits.score(i); if (!binningFilter.isEmpty()) { writeRecord = recordInBin(doc, binningFilter, String.valueOf(score)); } if (writeRecord) { itemCount++; if (doc.get("identifier") != null) temp = "\n"); else temp = "\n"; tempResult = tempResult.append(temp); String[] scoreArray = scoreText.split("\n"); for (int j=0; j"+scoreValue.trim()+"\n"; } else { if (doc.get(key) != null) { tempScore = ""+(doc.get(key)).trim()+"\n"; } else { tempScore = "null\n"; } } tempResult = tempResult.append(tempScore); } else tempResult = tempResult.append(scoreArray[j]); } tempResult = tempResult.append("\n"); String fieldValue = ""; for (Enumeration docField = doc.fields(); docField.hasMoreElements(); ) { Object a = docField.nextElement(); String data = a.toString(); int pos1 = data.indexOf('<'); int pos2 = data.indexOf(':'); String field = data.substring(pos1+1, pos2); if (!field.equals("contents")) { tempResult = tempResult.append("\n\t<"); tempResult = tempResult.append(field); tempResult = tempResult.append(">"); if (doc.get(field) != null) fieldValue = (doc.get(field)).trim(); if (field.equals("date")) { fieldValue = formatDate(fieldValue); } tempResult = tempResult.append(fieldValue); tempResult = tempResult.append(""); } } tempResult = tempResult.append("\n"); tempResult = tempResult.append("\n"); } } result = result.append("\n"); result = result.append(tempResult); result = result.append("\n"); } catch (IOException e) { System.out.println("Error --" + e.toString()); } return itemCount; } /** main method of this class to start binning process * @param * @returns result formatted in XML in StringBuffer * */ public StringBuffer processOrgSpec1D(Hits hits, ArrayList dimList) { String binningInfo = ""; ArrayList binningFilter = new ArrayList(); String scoreText = ""; StringBuffer result = new StringBuffer(); binningInfo = (String)dimList.get(ORGSPECS_BIN_INDEX); if (binningInfo.startsWith("trivial")) { processTrivialBinningDim(hits, dimList, binningFilter, scoreText, result); } else if (binningInfo.startsWith("natural")) { processNaturalBinningDim(hits, dimList, binningFilter, scoreText, result); } else if (binningInfo.startsWith("fixed")) { processFixedBinningDim(hits, dimList, binningFilter, scoreText, result); } return result; } /** add derived key to the result set * @param * @returns updated result set * */ public Hits addKeyToHits(Hits hits, String keyInfo, String keyName) { try{ if (keyInfo.startsWith("combination")) { ArrayList metadata = new ArrayList(); Metadata mdata; ProcessKey key = new ProcessKey(); metadata = key.getMetadata(keyInfo); //dim name is the key name int length = 0; if (hits.length() > HITS_LIMIT_INDEX) length = HITS_LIMIT_INDEX; else length = hits.length(); for (int i = 0; i < length; i++) { Document doc = hits.doc(i); for (int k=0; k 0) && (sStringAnalyze.charAt(nIndex-1) == '-')) { sSearchTerm = sStringAnalyze.substring(nIndex + 9, nNextIndex-1); arlOutContent.add(sSearchTerm); } else { sSearchTerm = sStringAnalyze.substring(nIndex + 9, nNextIndex-1); arlInContent.add(sSearchTerm); } nIndex = nNextIndex; nStart = nIndex; } } } public void printArrayListAsString(ArrayList arlList) { if ((arlList == null) || (arlList.size() <=0)) { System.out.println("Empty arraylist. Cannot Print"); } for (int arCounter = 0; arCounter < arlList.size(); arCounter++) { System.out.println(arCounter + ":" + (String) arlList.get(arCounter)); } } /* public static void main(String[] args) { TestBed testBed = new TestBed(); System.out.println("Parsing: " + sCommandLine); testBed.GetContentTerms(sCommandLine); System.out.println("--- Include Documents with "); testBed.printArrayListAsString(testBed.arlInContent); System.out.println("--- exclude Documents with "); testBed.printArrayListAsString(testBed.arlOutContent); } */ } qm-search-v0.5/src/search/ProcessDimTagDLConfig.java0000775000077000007700000001632310377650011022301 0ustar jfentonjfenton/* * ProcessDimTagDLConfig.java * Class to read Dimension tag from DL config file */ import java.io.*; import java.util.ArrayList; import org.w3c.dom.*; import org.xml.sax.*; import javax.xml.parsers.*; public class ProcessDimTagDLConfig { private static String keyString = ""; private ArrayList orgSpecList = new ArrayList(); private static ArrayList orgSpecList2D = new ArrayList(); /** Creates a new instance of ParseXMLFile */ public ProcessDimTagDLConfig(String profile) { // get the configured DL's xml file String xmlFileName = profile; // parse XML file -> XML document will be build Document doc = parseFile(xmlFileName); // get root node of xml tree structure if (doc != null) { Node root = doc.getDocumentElement(); writeDocumentToOutput(root); } } /** Returns element value * @param elem element (it is XML tag) * @return Element value otherwise empty String */ public final static String getElementValue( Node elem ) { Node kid; if( elem != null){ if (elem.hasChildNodes()){ for( kid = elem.getFirstChild(); kid != null; kid = kid.getNextSibling() ){ if( kid.getNodeType() == Node.TEXT_NODE ){ return kid.getNodeValue(); } } } } return ""; } /** Writes node and all child nodes into System.out * @param node XML node from XML tree from which parsing will start */ public void writeDocumentToOutput(Node node) { // write all child nodes recursively NodeList children = node.getChildNodes(); for (int i = 0; i < children.getLength(); i++) { Node child = children.item(i); if ((child.getNodeType() == Node.ELEMENT_NODE)&&(child.getNodeName() == "dim")) { //Process dimension tag processDimTag(child); orgSpecList2D.add(orgSpecList); orgSpecList = new ArrayList(); } } } /** process dim tag of XML file */ public void processDimTag(Node node) { NamedNodeMap attributes = node.getAttributes(); String name = "", dimString = ""; Node attribute = attributes.getNamedItem("name"); name = attribute.getNodeValue(); dimString=name; orgSpecList.add(dimString); // write all child nodes recursively NodeList children = node.getChildNodes(); for (int i = 0; i < children.getLength(); i++) { Node child = children.item(i); if (child.getNodeType() == Node.ELEMENT_NODE) { if (child.getNodeName().equals("key")) { processKeyTag(child); } if (child.getNodeName().equals("dim")) { processDimTag(child); } if (child.getNodeName().equals("binning")) { processBinningTag(child); } } } } /** process binning tag of XML file */ public void processBinningTag(Node node) { String binningString = ""; NamedNodeMap attributes = node.getAttributes(); if (attributes.getLength() == 0) { //System.out.println("\t\tno binning"); binningString = binningString.concat("no binning"); } else { for (int i = 0; i < attributes.getLength(); i++) { Node attribute = attributes.getNamedItem("type"); binningString = binningString.concat(attribute.getNodeValue()).concat(" "); //System.out.println("\t\t"+attribute.getNodeName()+"="+attribute.getNodeValue()); } //System.out.println("\t\tvalue = "+ getElementValue(node)); binningString = binningString.concat(getElementValue(node)); } orgSpecList.add(binningString); } /** process key tag of XML file */ public void processKeyTag(Node node) { keyString = ""; NamedNodeMap attributes = node.getAttributes(); String keyName = ""; if (attributes.getLength() == 0) { } else { Node attribute = attributes.getNamedItem("name"); keyName = attribute.getNodeValue(); //System.out.println("hi = "+keyName); } NodeList children = node.getChildNodes(); for (int i = 0; i < children.getLength(); i++) { Node child = children.item(i); if (child.getNodeType()== Node.ELEMENT_NODE) { if (child.getNodeName().equalsIgnoreCase("metadata")) { keyString = getElementValue(child); } else if (child.getNodeName().equalsIgnoreCase("combination")) { if (keyName != "") { keyString = "combination:".concat(keyName.trim()).concat(" "); } else keyString = "combination:"; processCombinationTag(child); } } } orgSpecList.add(keyString); } /** process combination tag of XML file */ public void processCombinationTag(Node node) { Node attribute; NamedNodeMap attributes = node.getAttributes(); if (attributes.getLength() == 0) { //System.out.println("\t\tno function name"); } else { for (int i = 0; i < attributes.getLength(); i++) { attribute = attributes.getNamedItem("function"); //System.out.println("\t\t"+attribute.getNodeName()+" = "+attribute.getNodeValue()); keyString = keyString.concat(attribute.getNodeValue()); } NodeList children = node.getChildNodes(); for (int i = 0; i < children.getLength(); i++) { Node child = children.item(i); if (child.getNodeType()== Node.ELEMENT_NODE) { attributes = child.getAttributes(); attribute = attributes.getNamedItem("metadata"); //System.out.println("\t\t\t"+attribute.getNodeName()+" = "+attribute.getNodeValue()); keyString = keyString.concat(" ").concat(attribute.getNodeValue()); attribute = attributes.getNamedItem("weight"); keyString = keyString.concat("=").concat(attribute.getNodeValue()); } } } } /** Parses XML file and returns XML document. * @param fileName XML file to parse * @return XML document or null if error occured */ public Document parseFile(String fileName) { DocumentBuilder docBuilder; Document doc = null; DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringElementContentWhitespace(true); try { docBuilder = docBuilderFactory.newDocumentBuilder(); } catch (ParserConfigurationException e) { System.out.println("Wrong parser configuration: " + e.getMessage()); return null; } File sourceFile = new File(fileName); try { doc = docBuilder.parse(sourceFile); } catch (SAXException e) { System.out.println("Wrong XML file structure: " + e.getMessage()); return null; } catch (IOException e) { System.out.println("Could not read source file: " + e.getMessage()); } return doc; } /** Starts XML parsing */ public ArrayList getOrgInfo() { return orgSpecList2D; } /* public static void main(String args[]) { ArrayList data = new ArrayList() data = getOrgInfo(); display2DArraylist(data); } */ } qm-search-v0.5/src/search/ProcessFilter.java0000644000077000007700000000473510403100662021002 0ustar jfentonjfenton/********************************************************************* * ProcessFilter.java * Class to process filtering to filter search results * Input: * Authur: * Urvashi Gadi (ugadi@emory.edu) * Date: * 10/2/2005 *******************************************************************/ import java.util.ArrayList; import org.apache.lucene.search.*; import org.apache.lucene.index.Term; public class ProcessFilter{ //NOT (level range 0.6 to 0.9) public static FilterData getFilterBreakup(String filterString) { FilterData filterInfo = new FilterData(); String filterSubstring = ""; if (filterString.startsWith("NOT")) { filterInfo.isExcluded = true; filterSubstring = filterString.substring(5, filterString.length()-1); } else { filterInfo.isExcluded = false; filterSubstring = filterString.substring(1, filterString.length()-1); } String temp[] = filterSubstring.split(" "); if (temp[0] != "") filterInfo.metadata = temp[0]; if (temp[1] != "") filterInfo.method = temp[1]; if (filterInfo.method.equals("range")) { filterInfo.lowerVal = temp[2]; //temp 3 is "to" filterInfo.upperVal = temp[4]; } else if (filterInfo.method.equals("<")) { filterInfo.upperVal = temp[2]; } else if (filterInfo.method.equals(">")) { filterInfo.lowerVal = temp[2]; } // for = case else if (filterInfo.method.equals("=")) { filterInfo.lowerVal = temp[2]; } return filterInfo; } public BooleanQuery getFilter(String f) { String filterString = "(level range 0.6 to 0.9)"; FilterData filterInfo; BooleanQuery filter = new BooleanQuery(); String filters[] = filterString.split("AND|OR"); for (int i=0; i XML document will be build Document doc = parseFile(xmlFileName); // get root node of xml tree structure if (doc != null) { Node root = doc.getDocumentElement(); writeDocumentToOutput(root); } } /** Returns element value * @param elem element (it is XML tag) * @return Element value otherwise empty String */ public final static String getElementValue( Node elem ) { Node kid; if( elem != null){ if (elem.hasChildNodes()){ for( kid = elem.getFirstChild(); kid != null; kid = kid.getNextSibling() ){ if( kid.getNodeType() == Node.TEXT_NODE ){ return kid.getNodeValue(); } } } } return ""; } /** @param node XML node from XML tree from which will output statement start * @param indent number of spaces used to indent output */ public void writeDocumentToOutput(Node node) { // write all child nodes recursively NodeList children = node.getChildNodes(); for (int i = 0; i < children.getLength(); i++) { Node child = children.item(i); if ((child.getNodeType() == Node.ELEMENT_NODE)&&(child.getNodeName() == "filter")) { //Process filter tag processFilterTag(child); } } } /** process filter tag from dl config */ public void processFilterTag(Node node) { // write all child nodes recursively boolean isNodeProcessed = false; NodeList children = node.getChildNodes(); for (int i = 0; i < children.getLength() && !isNodeProcessed; i++) { Node child = children.item(i); if (child.getNodeType() == Node.ELEMENT_NODE) { if (child.getNodeName().equals("fil_and")) { processBooleanClause(child, "AND", "fil_and"); isNodeProcessed = true; } if (child.getNodeName().equals("fil_or")) { processBooleanClause(child, "OR", "fil_or"); isNodeProcessed = true; } if (child.getNodeName().equals("clause")) { processNodeClause(child); isNodeProcessed = true; } } } if (children.getLength() > 0) { dlQuery = dlQuery.concat(")"); } } /** process boolean clause from dl config */ public void processBooleanClause(Node node, String sBooleanClause, String sBooleanTag) { if (node == null) {return;} if ((node.getNodeType() != Node.ELEMENT_NODE) || !(node.getNodeName().equalsIgnoreCase(sBooleanTag)) ) { return; } ArrayList listElementsNodes; int elementCount = 0; listElementsNodes = new ArrayList(); NodeList children = node.getChildNodes(); for (int i = 0; i < children.getLength(); i++) { Node child = children.item(i); if (child.getNodeType()== Node.ELEMENT_NODE) { listElementsNodes.add(child); } } elementCount = listElementsNodes.size(); if (elementCount > 0) { dlQuery = dlQuery.concat("("); for (int i = 0; i < elementCount; i++) { boolean wasNodeProcessed = false; String childNodeName; Node child = (Node)listElementsNodes.get(i); childNodeName = child.getNodeName(); if (childNodeName.equalsIgnoreCase("clause")) { processNodeClause(child); wasNodeProcessed = true; } if (childNodeName.equalsIgnoreCase("fil_or")) { processBooleanClause(child, "OR", "fil_or"); wasNodeProcessed = true; } if (childNodeName.equalsIgnoreCase("fil_and")) { processBooleanClause(child, "AND", "fil_and"); wasNodeProcessed = true; } if (wasNodeProcessed) { if (i < elementCount-1) { dlQuery = dlQuery.concat(") " + sBooleanClause + " "); } else { dlQuery = dlQuery.concat(")"); } } } } } /** Parses XML file and returns XML document. * @param fileName XML file to parse * @return XML document or null if error occured */ public Document parseFile(String fileName) { DocumentBuilder docBuilder; Document doc = null; DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringElementContentWhitespace(true); try { docBuilder = docBuilderFactory.newDocumentBuilder(); } catch (ParserConfigurationException e) { System.out.println("Wrong parser configuration: " + e.getMessage()); return null; } File sourceFile = new File(fileName); try { doc = docBuilder.parse(sourceFile); } catch (SAXException e) { System.out.println("Wrong XML file structure: " + e.getMessage()); return null; } catch (IOException e) { System.out.println("Could not read source file: " + e.getMessage()); } return doc; } /** process node clause from dl config */ public void processNodeClause(Node clauseNode) { if (clauseNode != null) { if ((clauseNode.getNodeType() == Node.ELEMENT_NODE) && (clauseNode.getNodeName().equalsIgnoreCase("clause"))) { FilterClause filterClause = new FilterClause(); dlQuery = dlQuery.concat("("); NamedNodeMap childAttributes = clauseNode.getAttributes(); for (int i = 0; i "); } File indexDir = new File(args[0]); //id is used the create a random result dir to store search results int id = Integer.parseInt(args[1]); String profile = args[2]; //read complete user query from command line String userQuery = ""; for (int i=3; i0) { //put everything after : in double quotes int index = args[i].indexOf(":"); String term = args[i].substring(0, index); term = term.concat(":\"").concat(args[i].substring(index+1, args[i].length())).concat("\""); userQuery = userQuery.concat(" ").concat(term); } else { userQuery = userQuery.concat(" \""); userQuery = userQuery.concat(args[i]).concat("\""); } } // System.out.println("query = "+userQuery); //read DL filter preference String dlQuery = ""; ProcessFilterTagDLConfig processDLConfig = new ProcessFilterTagDLConfig(profile); dlQuery = processDLConfig.getDlQuery(); //System.out.println("dl query = "+dlQuery); if (!indexDir.exists() || !indexDir.isDirectory()) { throw new Exception(indexDir + " does not exist or is not a directory."); } // call search method with complete query Searcher searchingApp = new Searcher(); searchingApp.search(indexDir, id, userQuery, dlQuery, profile); } /**Opens index, parses query, searches index and generates search result * @param Index Directory * @param query * @param search request id * @param specifications */ public void search(File indexDir, int id, String userQuery, String dlQuery, String profile) throws Exception{ Directory fsDir = FSDirectory.getDirectory(indexDir, false); IndexSearcher is = new IndexSearcher(fsDir); Hits hits; if (dlQuery.equals("")) { System.out.println(userQuery); Query query = QueryParser.parse(userQuery, "contents", new StandardAnalyzer()); System.out.println(query); hits = is.search(query); //System.out.println("Found " + hits.length() + " document(s) that matched query '" + userQuery); processOrgSpecifications(hits, id, userQuery, dlQuery, profile); } else { //filter constraint ProcessFilter processFilter = new ProcessFilter(); FilterData filterInfo = processFilter.getFilterBreakup(dlQuery); if (filterInfo.method.equals("range")) { RangeFilter rFilter = new RangeFilter(filterInfo.metadata, filterInfo.lowerVal, filterInfo.upperVal, true, true); String q = userQuery; Query query = QueryParser.parse(q, "contents", new StandardAnalyzer()); hits = is.search(query, rFilter); processOrgSpecifications(hits, id, userQuery, dlQuery, profile); } else if (filterInfo.method.equals("<")) { RangeFilter lessFilter = RangeFilter.Less(filterInfo.metadata, filterInfo.upperVal); String q = userQuery; Query query = QueryParser.parse(q, "contents", new StandardAnalyzer()); hits = is.search(query, lessFilter); processOrgSpecifications(hits, id, userQuery, dlQuery, profile); } else if (filterInfo.method.equals(">")) { RangeFilter moreFilter = RangeFilter.More(filterInfo.metadata, filterInfo.lowerVal); String q = userQuery; Query query = QueryParser.parse(q, "contents", new StandardAnalyzer()); hits = is.search(query, moreFilter); processOrgSpecifications(hits, id, userQuery, dlQuery, profile); } else if (filterInfo.method.equals("=")) { Query query = QueryParser.parse(userQuery, "contents", new StandardAnalyzer()); BooleanQuery q = new BooleanQuery(); q.add(query, BooleanClause.Occur.MUST); TermQuery tQuery = new TermQuery(new Term(filterInfo.metadata, filterInfo.lowerVal)); if (filterInfo.isExcluded) { q.add(tQuery, BooleanClause.Occur.MUST_NOT); } else q.add(tQuery, BooleanClause.Occur.MUST); hits = is.search(q); processOrgSpecifications(hits, id, userQuery, dlQuery, profile); } else { System.out.println("Filter : "+filterInfo.method+" not supported!"); } } //System.out.println("Found " + hits.length() + " document(s) that matched query '" + q + "':"); } /** Utility function to create a directory if it doesn't exist already * @param directory name to created */ public void createDir(String dirName) { //create dir to store output files File directory = new File(dirName); if ( ! directory.exists() ) { //System.out.println("Specified directory does not exist."); String path = (directory.getAbsolutePath()); boolean success = (new File(path)).mkdir(); if (!success) { System.out.println("dir "+dirName+" could not be created"); } } } /** creates output file based on organization specifications * @param hits * @param user query * @param dl query * @param search request id * @param specification */ public void processOrgSpecifications (Hits hits, int id, String userQuery, String dlQuery, String profile) throws Exception { StringBuffer result = new StringBuffer(); result = result.append("\n"); //get the style sheet name and update string buffer int sIndex = profile.lastIndexOf("/"); int eIndex = profile.lastIndexOf("."); String xslFile = XSLT_PATH.concat(profile.substring(sIndex, eIndex)).concat(".xsl"); result.append("\n"); result = result.append("\n"); String temp=""; if (dlQuery != "") temp = "\n"; else temp = "\n"; result = result.append(temp); //get search terms result = result.append(getTermsForXmlOutput(userQuery)); result = result.append("\n"); if (hits.length() > 0) { //read and process org specifications ProcessDimTagDLConfig processDimConfig = new ProcessDimTagDLConfig(profile); ArrayList orgSpecList, dimList; orgSpecList = processDimConfig.getOrgInfo(); for (int i = 0; i < orgSpecList.size(); i++) { dimList = (ArrayList) orgSpecList.get(i); ProcessDim processOrgSpecDim = new ProcessDim(); result = result.append(processOrgSpecDim.processOrgSpec1D(hits, dimList)); } } result = result.append("\n"); // create result dir if it does not exist already createDir("result"); String outputDirname = "result/result".concat(String.valueOf(id)); createDir(outputDirname); String outputFilename = outputDirname.concat("/output.xml"); FileOutputStream file = new FileOutputStream(outputFilename); file.write(result.toString().getBytes()); } /** Utility function to get terms out of user query * @param userQuery * @returns terms with XML tags */ public String getTermsForXmlOutput(String userQuery) { String terms = ""; try { Query query = QueryParser.parse(userQuery, "contents", new StandardAnalyzer()); GetTermsFromQuery getTerms = new GetTermsFromQuery(); getTerms.GetContentTerms(query.toString()); ArrayList arlList = getTerms.arlInContent; for (int arCounter = 0; arCounter < arlList.size(); arCounter++) { String term = (String)arlList.get(arCounter); term = term.replaceAll("\"", ""); terms = terms.concat(""+term.trim()+"\n"); } } catch (ParseException e) { System.out.println("Error --" + e.toString()); } return terms; } } qm-search-v0.5/src/search/result/0000775000077000007700000000000010405361555016676 5ustar jfentonjfentonqm-search-v0.5/src/search/Metadata.java0000775000077000007700000000026510401655713017747 0ustar jfentonjfenton/* this class is used to store metadata information for key calculation in * org specs */ public class Metadata{ public String name; public String value; public String wt; } qm-search-v0.5/src/search/FilterClause.java0000775000077000007700000000131510401131151020567 0ustar jfentonjfenton/******************************************************************** * filterClause.java * Helper class to read DL's filter tag from XML file * Authur: * Urvashi Gadi (ugadi@emory.edu) * Date: * 10/2/2005 * ******************************************************************/ public class FilterClause { public String metadata; public String function; public String value; public boolean isExcluded; public FilterClause() { metadata = ""; function = ""; value = ""; isExcluded = false; } public FilterClause(String mdata, String func, String val, boolean isEx) { metadata = mdata; function = func; value = val; isExcluded = isEx; } } qm-search-v0.5/webApp/0000755000077000007700000000000010405355377014545 5ustar jfentonjfentonqm-search-v0.5/webApp/profiles/0000775000077000007700000000000010405322713016356 5ustar jfentonjfentonqm-search-v0.5/webApp/profiles/spec3.xml0000755000077000007700000000100210377650011020112 0ustar jfentonjfenton score hits qm-search-v0.5/webApp/profiles/spec2.xml0000644000077000007700000000070410403331653020114 0ustar jfentonjfenton oaiset score qm-search-v0.5/webApp/profiles/spec4.xml0000644000077000007700000000115610403331773020123 0ustar jfentonjfenton oaiset hits 4 score qm-search-v0.5/webApp/profiles/profiles.txt0000644000077000007700000000013710377650011020744 0ustar jfentonjfentonspec1.xml:The CIC Portal spec2.xml:M9(A9-link) spec3.xml:Super-Google spec4.xml:Metascholar 3D qm-search-v0.5/webApp/profiles/spec1.xml0000755000077000007700000000044010377650011020115 0ustar jfentonjfenton granularity score qm-search-v0.5/webApp/xslts/0000775000077000007700000000000010403353255015713 5ustar jfentonjfentonqm-search-v0.5/webApp/xslts/spec2.xsl0000644000077000007700000002005510403336144017455 0ustar jfentonjfenton ]> 5 spec2 QM Search results for query: '<xsl:value-of select="query/@user"/>'
MetaScholar QM-Search (prototype)
—> results for query ''
Collection unknown ()
No results! Please use your browser's "back" button to try again.
.

128 (no description)
48
qm-search-v0.5/webApp/xslts/spec1.xsl0000644000077000007700000001555710403336144017467 0ustar jfentonjfenton ]> spec1 QM Search results for query: '<xsl:value-of select="query/@user"/>'
No results! Please use your browser's "back" button to try again.
MetaScholar QM-Search (prototype)
—> results for query ''
Results in Collections:
(no results)
Results in Individual Items:
(no results)
.

(no description)
qm-search-v0.5/webApp/xslts/common.xsl0000644000077000007700000003051510403336144017733 0ustar jfentonjfenton http://metascholar3.library.emory.edu/QMSearch/display.php http://metascholar3.library.emory.edu/QMSearch/profiles
New search:

Select view: radio profile spec1.xml checked CIC radio profile spec2.xml checked M9 radio profile spec3.xml checked S-G radio profile spec4.xml checked M3D

Change view:

Other tools:

  • view profile
  • (Go to "view page source" to see raw results XML)

Other links:

. 1 #### 0 ... , January February March April May June July August September October November December
subject:
type: () type: format:
(no date) , from qm-search-v0.5/webApp/xslts/spec3.xsl0000644000077000007700000001503210403336144017455 0ustar jfentonjfenton ]> spec3 QM Search results for query: '<xsl:value-of select="query/@user"/>'
MetaScholar QM-Search (prototype)
—> results for query ''
Sorted by both (combined): Sorted by popularity: Sorted by text similarity:
No results! Please use your browser's "back" button to try again.
.

(no description)
qm-search-v0.5/webApp/xslts/spec4.xsl0000644000077000007700000002606510403336144017466 0ustar jfentonjfenton ]> spec4 QM Search results for query: '<xsl:value-of select="query/@user"/>'
No results! Please use your browser's "back" button to try again.
MetaScholar QM-Search (prototype)
—> results for query ''
popularity
collection
(collection not known)
(no records in popularity level ) low medium high highest unknown . ( )
48 (no description) 32
qm-search-v0.5/webApp/display.php0000775000077000007700000000265510405355377016740 0ustar jfentonjfenton Result
Retrieving results....... qm-search-v0.5/webApp/index.php0000775000077000007700000000254210403354352016363 0ustar jfentonjfenton Search
'; echo ''; ?> Search for:

qm-search-v0.5/README0000644000077000007700000001331410405361412014174 0ustar jfentonjfenton QM-Search Framework version 0.5 2006-02-21 Urvashi Gadi Emory University Credits ======= Urvashi Gadi - Lead developer. Aaron Krowne - Project Manager. Overview ======== QM-Search framework is the outcome of a series of studies conducted at Emory University using production digital library services to determine the criteria that underlies the preferences and assumptions of different groups of users regarding metasearch systems. This system does not re-invent basic search functionality but uses open source Apache Lucene libraries. Various layers are added to the libraries to generalize filtering/organization across all metadata attributes, to allow ranking based on any number of quality indicators and to allow any number of indicators/fields to go into a results organization. The framework does not commit to any interface, but makes interfaces easy to generate. See http://metascholar.org/quality_metrics/ for more on the quality metrics project. The Model ========= QM-Search consists of only two core components: 1. the ranking/organizing system 2. the analysis system QM-Search allows you to organize and rank results in a generalized way based on metadata attributes or quality indicators. Both metadata attributes and quality indicators are stored as Lucene Document fields, so they are treated uniformly by the ranking/organizing system. The only distinction is that metadata attributes are "already there" and quality indicators are added by batch analysis/data mining by the analysis modules (in the analysis system). The system also consists of the auxiliary components: - an indexer - a presentation layer - (usually) a harvester The harvester is required in situations where you do not already have your records locally. The indexer is the standard Lucene provision for getting the data in to the index; you must write this. We suggest having it normalize and sanitize the metadata to provide a consistent experience for your users. The analyzers run after the indexer, and they add desired quality indicator fields to the indexed records. The presentation layer turns the QM-Search system's XML output into a user interface. This can be anything you like that takes in XML and interprets XML and outputs HTML. We provide XSL stylesheets to get the job done; you may opt for other methods. Installation ============ 1.Extract the source code to the directory of your choice on your file system. The archive contains a top level directory called qm-search-v0.5, so its safe to extract it to your home directory. 2.Under the created qm-framework-0.5 directory, you will find 4 directories: "src", "build", "harvesters" and "webApp". "src" directory contains indexing and searching code. "webApp" directory contains web access code to the search system. "harvestors" dir contains a few harvestors and sample data that can be indexed. "build" directory should contain Lucene build. Please download the jar file from http://www.apache.org/dyn/closer.cgi/lucene/java/. This code is tested for Lucene core version 1.9.1 In case you decide to download some other version of Lucene release from jakarta website (http://jakarta.apache.org), You might have to make changes to the code in "src" directory to make the framework work with the latest Lucene release. 3.Compile the index and search code using build file build.xml >ant -v (inside src directory). It requires Apache Ant to be configured on the machine. Running ======= Indexer ------- You may need to harvest data from some remote source. To this end, The "harvesters" directory (qm-framework-0.5/harvesters/) contains a few OAI harvesters in Perl and sample data in the format in which the included indexer expects the data to be in. You should write your own harvester (or modify ours) based on your data sources. After you have your data in a text file, you can index your data using the following command on the command line: java -cp .:$CLASSPATH IndexingApp where is the directory in which the index is created and is the text file containing the data to be indexed. You can also use the script file "runApp" located in src/index/ directory which requires above three parameters. Since indexing involves writing index files onto the disk, the process could be slow depending upon the size of the data being indexed. Searcher -------- You can search for any query in the indexed data using our web application included with the code (webApp/). It requires Apache web server as container so make sure that it is properly configured to execute PHP code. The web application provides a "standard" web-like search interface. The user also needs to select the profile in which he wants the result of search to be displayed in. The included profiles (XSLTs) map to common scenarios we have selected. You will probably want to customize these transforms or add new ones for your own search scenarios. Before you can execute any search request, you will have to change the result and xslt path in search code (search/) and execute path in index.php and display.php(/webApp) according to the location of the code on your file system. Change Logs =========== 0.5 - First release. Development Roadmap =================== 0.75 - Analysis Plugins for "data mining" 1.0 - Org. spec. editor License ======= BSD. See included "LICENSE" file. qm-search-v0.5/LICENSE0000644000077000007700000000134210377654172014337 0ustar jfentonjfentonCopyright (c) 2006, Urvashi Gadi, Emory University All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Emory University nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.