PHP function to get the most recurrent words in a file - it uses Linux command line
August 3rd, 2010 /**
* Extracts the most recurrent one-word and two-word terms in a file
* Filters out some common stop words and you can also pass extra ones
*
* @param string $filepath
* @param int $minWordLength - the minimal word length for the terms to extract
* @param int $numberOfTerms - the number of terms to retrieve
* @return array of string - the most recurrent terms
*/
function getMostRecurrentTermsInFile ($filepath, $minWordLength, $numberOfTerms, array $extraStopWords)
{
$stopwords = array(’a', ‘about’, ‘above’, ‘above’, ‘across’, ‘after’, ‘afterwards’, ‘again’, ‘against’, ‘all’,
‘almost’, ‘alone’, ‘along’, ‘already’, ‘also’,'although’,'always’,'am’,'among’, ‘amongst’, ‘amoungst’, ‘amount’,
‘an’, ‘and’, ‘another’, ‘any’,'anyhow’,'anyone’,'anything’,'anyway’, ‘anywhere’, ‘are’, ‘around’, ‘as’, ‘at’,
‘back’,'be’,'became’, ‘because’,'become’,'becomes’, ‘becoming’, ‘been’, ‘before’, ‘beforehand’, ‘behind’, ‘being’,
‘below’, ‘beside’, ‘besides’, ‘between’, ‘beyond’, ‘bill’, ‘both’, ‘bottom’,'but’, ‘by’, ‘call’, ‘can’, ‘cannot’,
‘cant’, ‘co’, ‘con’, ‘could’, ‘couldnt’, ‘cry’, ‘de’, ‘describe’, ‘detail’, ‘do’, ‘done’, ‘down’, ‘due’, ‘during’,
‘each’, ‘eg’, ‘eight’, ‘either’, ‘eleven’,'else’, ‘elsewhere’, ‘empty’, ‘enough’, ‘etc’, ‘even’, ‘ever’, ‘every’,
‘everyone’, ‘everything’, ‘everywhere’, ‘except’, ‘few’, ‘fifteen’, ‘fify’, ‘fill’, ‘find’, ‘fire’, ‘first’, ‘five’,
‘for’, ‘former’, ‘formerly’, ‘forty’, ‘found’, ‘four’, ‘from’, ‘front’, ‘full’, ‘further’, ‘get’, ‘give’, ‘go’, ‘had’,
‘has’, ‘hasnt’, ‘have’, ‘he’, ‘hence’, ‘her’, ‘here’, ‘hereafter’, ‘hereby’, ‘herein’, ‘hereupon’, ‘hers’, ‘herself’,
‘him’, ‘himself’, ‘his’, ‘how’, ‘however’, ‘hundred’, ‘ie’, ‘if’, ‘in’, ‘inc’, ‘indeed’, ‘interest’, ‘into’, ‘is’, ‘it’,
‘its’, ‘itself’, ‘keep’, ‘last’, ‘latter’, ‘latterly’, ‘least’, ‘less’, ‘ltd’, ‘made’, ‘many’, ‘may’, ‘me’, ‘meanwhile’,
‘might’, ‘mill’, ‘mine’, ‘more’, ‘moreover’, ‘most’, ‘mostly’, ‘move’, ‘much’, ‘must’, ‘my’, ‘myself’, ‘name’, ‘namely’,
‘neither’, ‘never’, ‘nevertheless’, ‘next’, ‘nine’, ‘no’, ‘nobody’, ‘none’, ‘noone’, ‘nor’, ‘not’, ‘nothing’, ‘now’,
‘nowhere’, ‘of’, ‘off’, ‘often’, ‘on’, ‘once’, ‘one’, ‘only’, ‘onto’, ‘or’, ‘other’, ‘others’, ‘otherwise’, ‘our’, ‘ours’,
‘ourselves’, ‘out’, ‘over’, ‘own’,'part’, ‘per’, ‘perhaps’, ‘please’, ‘put’, ‘rather’, ‘re’, ’same’, ’see’, ’seem’, ’seemed’,
’seeming’, ’seems’, ’serious’, ’several’, ’she’, ’should’, ’show’, ’side’, ’since’, ’sincere’, ’six’, ’sixty’, ’so’, ’some’,
’somehow’, ’someone’, ’something’, ’sometime’, ’sometimes’, ’somewhere’, ’still’, ’such’, ’system’, ‘take’, ‘ten’, ‘than’,
‘that’, ‘the’, ‘their’, ‘them’, ‘themselves’, ‘then’, ‘thence’, ‘there’, ‘thereafter’, ‘thereby’, ‘therefore’, ‘therein’,
‘thereupon’, ‘these’, ‘they’, ‘thickv’, ‘thin’, ‘third’, ‘this’, ‘those’, ‘though’, ‘three’, ‘through’, ‘throughout’, ‘thru’,
‘thus’, ‘to’, ‘together’, ‘too’, ‘top’, ‘toward’, ‘towards’, ‘twelve’, ‘twenty’, ‘two’, ‘un’, ‘under’, ‘until’, ‘up’, ‘upon’,
‘us’, ‘very’, ‘via’, ‘was’, ‘we’, ‘well’, ‘were’, ‘what’, ‘whatever’, ‘when’, ‘whence’, ‘whenever’, ‘where’, ‘whereafter’,
‘whereas’, ‘whereby’, ‘wherein’, ‘whereupon’, ‘wherever’, ‘whether’, ‘which’, ‘while’, ‘whither’, ‘who’, ‘whoever’, ‘whole’,
‘whom’, ‘whose’, ‘why’, ‘will’, ‘with’, ‘within’, ‘without’, ‘would’, ‘yet’, ‘you’, ‘your’, ‘yours’, ‘yourself’, ‘yourselves’,
‘the’);
$stopwords = array_merge($stopwords, $extraStopWords);
// placing each word on a separate line
$command = ’sed -e “s/[^a-zA-Z]/\n/g” ‘ . $filepath;
$command .= ‘|’;
// striping out the empty lines
$command .= ‘grep -v “^$”‘;
$command .= ‘|’;
// adding lines combining all adjacent two words
// N.B.: I am commenting the single quotes inside the command
$command .= ‘awk \’(PREV!=”") {printf “%s\n%s %s\n”, PREV, PREV, $1} {PREV=$1}\”;
$command .= ‘|’;
// stripping out common stopwords (the actual command is something like this: grep -Ev ‘(\bis\b|\bsuch\b)’)
$command .= ‘grep -Evi “(\b’;
$command .= implode(’\b|\b’, $stopwords);
$command .= ‘\b)”‘;
$command .= ‘|’;
// removing all the words shorter than $minWordLength characters
$limit = $minWordLength -1;
$command .= “grep -Ev ‘^[a-zA-Z]{1,$limit}$’”;
$command .= ‘|’;
// N.B.: we are commenting the single quotes inside the command
$command .= ’sort | uniq -c | sort -nr’;
$command .= ‘|’;
// stripping out the numbers we use for sorting
$command .= “sed -e ’s/^[^0-9]*[0-9]* //g’”;
$command .= ‘|’;
$command .= ” head -n $numberOfTerms”;
$commandOutput = shell_exec($command);
$commandOutputLines = explode(”\n”, $commandOutput);
// sanitising the return
$ret = array();
foreach ($commandOutputLines as $commandOutputLine)
{
$commandOutputLine = trim($commandOutputLine);
if ( strlen($commandOutputLine) > 0 )
{
$ret[] = $commandOutputLine;
}
}
return $ret;
}
