public class FileManglerTokenizer extends Tokenizer
tokenVectorMap
Constructor and Description |
---|
FileManglerTokenizer()
Constructor that initializes the random number generator and clears the mangler settings
|
FileManglerTokenizer(java.lang.String manglers)
Constructor that sets the mangler settings
|
Modifier and Type | Method and Description |
---|---|
void |
changePuncHelper(java.util.Vector<java.lang.Character> punc,
int start,
int end) |
void |
changeToChar(java.util.Vector<Token> newTokenVector) |
void |
changeToken(java.util.Vector<Token> newTokenVector) |
void |
changeToPunc(java.util.Vector<Token> newTokenVector) |
void |
deleteChar(java.util.Vector<Token> newVectorToken) |
void |
deleteHead() |
void |
deleteSection() |
void |
deleteTail() |
void |
deleteToken() |
void |
deleteWhiteSpace(java.util.Vector<Token> newTokenVector) |
void |
disableMangler()
Disables the mangler in this tokenizer
|
void |
enableMangler(java.lang.String manglers,
java.util.List<Token> tokenList)
Enables the mangler with the given settings in this tokenizer
|
void |
setRNG(java.util.Random random)
Sets the random number generator to use with the manglers that are set in this tokenizer
|
void |
splitToken(java.util.Vector<Token> newTokenVector) |
void |
summaryReduce(java.util.Vector<Token> newtokenVector) |
java.util.List<Token> |
tokenize(java.util.List<Token> tokens)
Alters or eliminates certain tokens based on the given mangler settings
|
void |
validateMangler(java.lang.String mangler,
float percent)
Ensures that the parameters provided to the mangler are correctly formatted
|
getTokenVectorMap, iterator, printTokens, tokenize, toString
public FileManglerTokenizer()
public FileManglerTokenizer(java.lang.String manglers)
manglers
- the string of mangler settings; expects non-null mangler strings in the form:
"manglerName-manglerPercent", where the manglerPercent is between 1 and 99 inclusivepublic void validateMangler(java.lang.String mangler, float percent)
mangler
- the current mangler that is going to be usedpercent
- the percent that is going to be mangledpublic void enableMangler(java.lang.String manglers, java.util.List<Token> tokenList)
manglers
- the string of mangler settings; expects non-null mangler strings in the form:
"manglerName-manglerPercent", where the manglerPercent is between 1 and 100 inclusivetokenList
- the list of string tokens to put through this manglerpublic void disableMangler()
public void setRNG(java.util.Random random)
random
- the random number generator to use with this tokenizerpublic java.util.List<Token> tokenize(java.util.List<Token> tokens)
public void deleteToken()
public void deleteSection()
public void deleteHead()
public void deleteTail()
public void deleteChar(java.util.Vector<Token> newVectorToken)
public void deleteWhiteSpace(java.util.Vector<Token> newTokenVector)
public void changeToChar(java.util.Vector<Token> newTokenVector)
public void splitToken(java.util.Vector<Token> newTokenVector)
public void changeToken(java.util.Vector<Token> newTokenVector)
public void changeToPunc(java.util.Vector<Token> newTokenVector)
public void changePuncHelper(java.util.Vector<java.lang.Character> punc, int start, int end)
public void summaryReduce(java.util.Vector<Token> newtokenVector)
newtokenVector
- passed by value