oracle.spatial.hadoop.vector.mapred.job

Class KMeansClustering<K,V>

java.lang.Object

org.apache.hadoop.conf.Configured

oracle.spatial.hadoop.vector.mapred.job.BaseJob<K,V>

oracle.spatial.hadoop.vector.mapred.job.KMeansClustering<K,V>

Type Parameters:

K - the type of the input keys

V - the type of the input values

All Implemented Interfaces:

org.apache.hadoop.conf.Configurable, org.apache.hadoop.util.Tool

Direct Known Subclasses:

KMeansClustering

public class KMeansClustering<K,V>
extends BaseJob<K,V>

Class used to perform spatial clustering over an input data set.The clustering is performed running one or more jobs iteratively.

Field Summary

Fields

Modifier and Type	Field and Description
static java.lang.String	ARG_CLUSTERS_POINTS
static java.lang.String	ARG_CRIT_FUN_CLASS
static java.lang.String	ARG_DELETE_CLUSTER_FILES
static java.lang.String	ARG_K
static java.lang.String	ARG_MAX_ITERATIONS
static java.lang.String	ARG_MAX_MEMBER_DISTANCE
static java.lang.String	ARG_SHAPE_GEN_CLASS
protected double[]	clustersPoints
protected java.lang.Class<? extends CriterionFunction>	critetionFunctionClass
protected boolean	deletePreviousClusterFiles
protected int	dimensions
protected org.apache.hadoop.fs.Path	inClustersPath
protected int	iteration
protected int	k
protected int	maxIterations
protected double	maxMemberDistance
protected java.lang.Class<? extends ClusterShapeGenerator>	shapeGeneratorClass
protected org.apache.hadoop.fs.Path	workDirPath

Fields inherited from class oracle.spatial.hadoop.vector.mapred.job.BaseJob

argsp, jarClass

Constructor Summary

Constructors

Constructor and Description
KMeansClustering()

Method Summary

Modifier and Type	Method and Description
void	configure(org.apache.hadoop.mapred.JobConf conf) Validates and adds the current parameters to the job configuration
protected void	deletePreviousClustersFiles(org.apache.hadoop.conf.Configuration conf)
protected void	generateFinalResultClustersFile(ClusterInfo[] clusters, org.apache.hadoop.conf.Configuration conf)
protected double[]	generateInitialKClustersPoints(org.apache.hadoop.conf.Configuration conf)
double[]	getClustersPoints() Gets the initial clusters points
java.lang.String	getCmdOptions() Gets a description of the arguments expected from command line.
java.lang.Class<? extends CriterionFunction>	getCritetionFunctionClass() Gets the CriterionFunction subclass to be used
org.apache.hadoop.fs.Path	getInClustersPath() Gets the path where the input cluster information is located
int	getIteration() Gets the current iteration number
int	getK() Gets the number of clusters
int	getMaxIterations() Gets the maximum number of iterations allowed
protected PointWritable[]	getSampledPoints(org.apache.hadoop.conf.Configuration conf)
java.lang.Class<? extends ClusterShapeGenerator>	getShapeGeneratorClass() Gets the ClusterShapeGenerator subclass to be used
org.apache.hadoop.fs.Path	getWorkDirPath() Gets the path where the mapreduce output is stored at each iteration
boolean	isDeletePreviousClusterFiles() Specifies whether the output generated by previous iterations should be removed or not
protected boolean	isNextIteration(int iteration, int convClusters, int maxIterations, java.lang.Class<? extends ClusterShapeGenerator> finalShapeGenClass)
static void	main(java.lang.String[] args)
protected static ClusterInfo[]	pointsToClusterInfo(double[] centers, int dimensions)
void	processArgs(java.lang.String[] args, org.apache.hadoop.conf.Configuration conf) Extracts and validates arguments from the command line
protected static void	replaceEmptyOutClusters(ClusterInfo[] outClusters, ClusterInfo[] inClusters)
int	run(java.lang.String[] args)
KMeans.KMeansIterationResult	runIteration(int iteration, org.apache.hadoop.mapred.JobConf baseConf) Launches a mapreduce job to run a single iteration
boolean	runIterations(int maxIterations, org.apache.hadoop.mapred.JobConf baseConf) Runs the K Means clustering algorithm using the given configuration
void	setClustersPoints(double[] clustersPoints) Sets the initial clusters points
void	setCritetionFunctionClass(java.lang.Class<? extends CriterionFunction> critetionFunctionClass) Sets the CriterionFunction subclass to be used
void	setDeletePreviousClusterFiles(boolean deletePreviousClusterFiles) Specifies whether the output generated by previous iterations should be removed or not
void	setInClustersPath(org.apache.hadoop.fs.Path inClustersPath) Sets the path where the input cluster information is located
void	setIteration(int iteration) Sets the current iteration number
void	setK(int k) Sets the number of clusters
void	setMaxIterations(int maxIterations) Sets the maximum number of iterations allowed
void	setOutput(java.lang.String output) Sets the job output
void	setShapeGeneratorClass(java.lang.Class<? extends ClusterShapeGenerator> shapeGeneratorClass) Sets the ClusterShapeGenerator subclass to be used
protected ClusterInfo[]	setupNextIteractionClusters(KMeans.KMeansIterationResult iterationResult, ClusterInfo[] previousIterationClusters, org.apache.hadoop.conf.Configuration conf)

Methods inherited from class oracle.spatial.hadoop.vector.mapred.job.BaseJob

configure, createJob, createJob, createJob, createJob, createJobConf, createJobConf, createJobConf, getInput, getInputFormatClass, getJarClass, getOutput, getRecordInfoProviderClass, getSpatialConfig, setInput, setInputFormatClass, setJarClass, setRecordInfoProviderClass, setSpatialConfig

Methods inherited from class org.apache.hadoop.conf.Configured

getConf, setConf

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.hadoop.conf.Configurable

getConf, setConf

Field Detail

k

protected int k

dimensions

protected int dimensions

iteration

protected int iteration

clustersPoints

protected double[] clustersPoints

inClustersPath

protected org.apache.hadoop.fs.Path inClustersPath

deletePreviousClusterFiles

protected boolean deletePreviousClusterFiles

maxIterations

protected int maxIterations

workDirPath

protected org.apache.hadoop.fs.Path workDirPath

critetionFunctionClass

protected java.lang.Class<? extends CriterionFunction> critetionFunctionClass

shapeGeneratorClass

protected java.lang.Class<? extends ClusterShapeGenerator> shapeGeneratorClass

maxMemberDistance

protected double maxMemberDistance

ARG_MAX_MEMBER_DISTANCE

public static final java.lang.String ARG_MAX_MEMBER_DISTANCE

See Also:

Constant Field Values

ARG_SHAPE_GEN_CLASS

public static final java.lang.String ARG_SHAPE_GEN_CLASS

See Also:

Constant Field Values

ARG_CRIT_FUN_CLASS

public static final java.lang.String ARG_CRIT_FUN_CLASS

See Also:

Constant Field Values

ARG_DELETE_CLUSTER_FILES

public static final java.lang.String ARG_DELETE_CLUSTER_FILES

See Also:

Constant Field Values

ARG_CLUSTERS_POINTS

public static final java.lang.String ARG_CLUSTERS_POINTS

See Also:

Constant Field Values

ARG_MAX_ITERATIONS

public static final java.lang.String ARG_MAX_ITERATIONS

See Also:

Constant Field Values

ARG_K

public static final java.lang.String ARG_K

See Also:

Constant Field Values

Constructor Detail

KMeansClustering

public KMeansClustering()

Method Detail

getWorkDirPath

public org.apache.hadoop.fs.Path getWorkDirPath()

Gets the path where the mapreduce output is stored at each iteration

Returns:

an HDFS path

setOutput

public void setOutput(java.lang.String output)

Description copied from class: BaseJob

Sets the job output

Overrides:

setOutput in class BaseJob<K,V>

Parameters:

output - a path

setClustersPoints

public void setClustersPoints(double[] clustersPoints)

Sets the initial clusters points

Parameters:

clustersPoints - an array of point ordinates in the form x1,y1,x2,y2,...,xK,yK

getClustersPoints

public double[] getClustersPoints()

Gets the initial clusters points

Returns:

an array of point ordinates in the form x1,y1,x2,y2,...,xK,yK

setInClustersPath

public void setInClustersPath(org.apache.hadoop.fs.Path inClustersPath)

Sets the path where the input cluster information is located

Parameters:

inClustersPath - a path

getInClustersPath

public org.apache.hadoop.fs.Path getInClustersPath()

Gets the path where the input cluster information is located

Returns:

a path

setK

public void setK(int k)

Sets the number of clusters

Parameters:

k - the number of clusters

getK

public int getK()

Gets the number of clusters

Returns:

the number of clusters

setIteration

public void setIteration(int iteration)

Sets the current iteration number

Parameters:

iteration - the current iteration number

getIteration

public int getIteration()

Gets the current iteration number

Returns:

the current iteration number

setDeletePreviousClusterFiles

public void setDeletePreviousClusterFiles(boolean deletePreviousClusterFiles)

Specifies whether the output generated by previous iterations should be removed or not

Parameters:

deletePreviousClusterFiles - true if intermediate output should be removed

isDeletePreviousClusterFiles

public boolean isDeletePreviousClusterFiles()

Specifies whether the output generated by previous iterations should be removed or not

Returns:

true if intermediate output should be removed

setMaxIterations

public void setMaxIterations(int maxIterations)

Sets the maximum number of iterations allowed

Parameters:

maxIterations - the maximum number of iterations allowed

getMaxIterations

public int getMaxIterations()

Gets the maximum number of iterations allowed

Returns:

the maximum number of iterations allowed

setCritetionFunctionClass

public void setCritetionFunctionClass(java.lang.Class<? extends CriterionFunction> critetionFunctionClass)

Sets the CriterionFunction subclass to be used

Parameters:

critetionFunctionClass - a CriterionFunction subclass

getCritetionFunctionClass

public java.lang.Class<? extends CriterionFunction> getCritetionFunctionClass()

Gets the CriterionFunction subclass to be used

Returns:

a CriterionFunction subclass

setShapeGeneratorClass

public void setShapeGeneratorClass(java.lang.Class<? extends ClusterShapeGenerator> shapeGeneratorClass)

Sets the ClusterShapeGenerator subclass to be used

Parameters:

shapeGeneratorClass - a ClusterShapeGenerator subclass

getShapeGeneratorClass

public java.lang.Class<? extends ClusterShapeGenerator> getShapeGeneratorClass()

Gets the ClusterShapeGenerator subclass to be used

Returns:

a ClusterShapeGenerator subclass

processArgs

public void processArgs(java.lang.String[] args,
                        org.apache.hadoop.conf.Configuration conf)
                 throws java.lang.Exception

Description copied from class: BaseJob

Extracts and validates arguments from the command line

Overrides:

processArgs in class BaseJob<K,V>

Parameters:

args - arguments from the command line

conf - the job configuration

Throws:

java.lang.Exception

getCmdOptions

public java.lang.String getCmdOptions()

Description copied from class: BaseJob

Gets a description of the arguments expected from command line.

Overrides:

getCmdOptions in class BaseJob<K,V>

Returns:

a text description

configure

public void configure(org.apache.hadoop.mapred.JobConf conf)
               throws java.lang.Exception

Description copied from class: BaseJob

Validates and adds the current parameters to the job configuration

Overrides:

configure in class BaseJob<K,V>

Parameters:

conf - the job configuration

Throws:

java.lang.Exception

runIteration

public KMeans.KMeansIterationResult runIteration(int iteration,
                                                 org.apache.hadoop.mapred.JobConf baseConf)
                                          throws java.lang.Exception

Launches a mapreduce job to run a single iteration

Parameters:

iteration - the iteration number

baseConf - the job configuration

Returns:

a KMeans.KMeansIterationResult containing the results of the iteration

Throws:

java.lang.Exception

runIterations

public boolean runIterations(int maxIterations,
                             org.apache.hadoop.mapred.JobConf baseConf)
                      throws java.lang.Exception

Runs the K Means clustering algorithm using the given configuration

Parameters:

maxIterations - the maximum number of iterations. If zero is passed, a default value based on the number of clusters will be used

baseConf - the job configuration

Returns:

true if the algortihm was successful

Throws:

java.lang.Exception

getSampledPoints

protected PointWritable[] getSampledPoints(org.apache.hadoop.conf.Configuration conf)
                                    throws java.io.IOException,
                                           java.lang.InterruptedException

Throws:

java.io.IOException

java.lang.InterruptedException

generateInitialKClustersPoints

protected double[] generateInitialKClustersPoints(org.apache.hadoop.conf.Configuration conf)
                                           throws java.lang.Exception

Throws:

java.lang.Exception

generateFinalResultClustersFile

protected void generateFinalResultClustersFile(ClusterInfo[] clusters,
                                               org.apache.hadoop.conf.Configuration conf)
                                        throws java.io.IOException

Throws:

java.io.IOException

deletePreviousClustersFiles

protected void deletePreviousClustersFiles(org.apache.hadoop.conf.Configuration conf)

isNextIteration

protected boolean isNextIteration(int iteration,
                                  int convClusters,
                                  int maxIterations,
                                  java.lang.Class<? extends ClusterShapeGenerator> finalShapeGenClass)

setupNextIteractionClusters

protected ClusterInfo[] setupNextIteractionClusters(KMeans.KMeansIterationResult iterationResult,
                                                    ClusterInfo[] previousIterationClusters,
                                                    org.apache.hadoop.conf.Configuration conf)
                                             throws java.io.IOException

Throws:

java.io.IOException

pointsToClusterInfo

protected static ClusterInfo[] pointsToClusterInfo(double[] centers,
                                                   int dimensions)

replaceEmptyOutClusters

protected static void replaceEmptyOutClusters(ClusterInfo[] outClusters,
                                              ClusterInfo[] inClusters)

run

public int run(java.lang.String[] args)
        throws java.lang.Exception

Throws:

java.lang.Exception

main

public static void main(java.lang.String[] args)
                 throws java.lang.Exception

Throws:

java.lang.Exception