Data classification (simple & naive)

#!/usr/bin/perl
use Modern::Perl;
no warnings;
no if $] >= 5.017011, warnings => 'experimental::smartmatch';

# do singleton check and possible bailout as early as possible.
use Fcntl qw(:flock);
my $lockfile = 'E:\\FlowData\\PreProcess\\classifier.lock';

sub BailOut {
	print "$0 is already running. Exiting.\n";
	exit(1);
}
open( my $fhpid, '>>', $lockfile ) or die "error: open '$lockfile': $!";
flock( $fhpid, LOCK_EX | LOCK_NB ) or BailOut();
use strict;
no warnings;
use AI::Categorizer;
use AI::Categorizer::Collection::Files;
use Algorithm::NaiveBayes::Model::Frequency;
use File::Spec;
use File::Copy::Vigilant qw(copy move);
use File::ChangeNotify;
use File::Basename;
our $cat_file = 'E:\\FlowData\\PreProcess\\cats.txt';
our $path     = 'E:\\FlowData\\Inbound';
my $watcher = File::ChangeNotify->instantiate_watcher(
	directories => [$path],
	filter      => qr/\.(?:xml|txt)$/,
	exclude => [ qr/(?:test|temp|audit|error)/is ],
);
use AI::Categorizer::Learner::NaiveBayes;
our %files;
our $nb = AI::Categorizer::Learner::NaiveBayes->restore_state('E:\\FlowData\\PreProcess\\state');
print "Begining to watch $path for changes.\n";

while ( my @events = $watcher->wait_for_events() ) {
	sleep(5);
	foreach my $event (@events) {
		if ( $event->type() eq "create" ) {
			sleep(2);
			my ( $filename, $dirs, $suffix ) = fileparse( $event->path() );
			print "Detected " . $event->type() . " on ".$filename."\n";
			if ( $filename =~ m/\.txt$/ ) {
				print "Moving assumed journal $filename\n";
				move( $dirs . $filename, "E:\\FlowData\\Journal\\" . $filename, check => 'md5', retries => 'infinite' );
			} else {
				open my $out_fh, '>', $cat_file;
				opendir( DIR, $path ) or die $!;
				while ( my $file = readdir(DIR) ) {

					# Use a regular expression to ignore files beginning with a period
					next if ( $file =~ m/^\./ );
					print {$out_fh} "$file \n";
				}
				closedir(DIR);
				close $out_fh;
				my $c = new AI::Categorizer::Collection::Files( path => $path, category_file => $cat_file );
				while ( my $document = $c->next ) {
					my $hypothesis = $nb->categorize($document);
					print "Classified ", $hypothesis->document_name();
					print " as ", $hypothesis->best_category, "\n";
					my $original   = $path . "\\" . $hypothesis->document_name();
					my $classified = 'E:\\FlowData\\PreProcess\\' . $hypothesis->best_category() . '\\' . $hypothesis->document_name();
					if ( $original =~ /^(.*)$/ ) {
						$original = $1;    # $data now untainted
					} else {
						die "Bad data in $original";    # log this somewhere
					}
					if ( $classified =~ /^(.*)$/ ) {
						$classified = $1;               # $data now untainted
					} else {
						die "Bad data in $classified";    # log this somewhere
					}
					move( $original, $classified, check => 'md5', retries => 'infinite' );
				}
			}
		}
	}
}

#!/usr/bin/perl

use Modern::Perl;

no warnings;

no if $] >= 5.017011, warnings => 'experimental::smartmatch';

# do singleton check and possible bailout as early as possible.

use Fcntl qw(:flock);

my $lockfile = 'E:\\FlowData\\PreProcess\\classifier.lock';

sub BailOut {

print "$0 is already running. Exiting.\n";

exit(1);

}

open( my $fhpid, '>>', $lockfile ) or die "error: open '$lockfile': $!";

flock( $fhpid, LOCK_EX | LOCK_NB ) or BailOut();

use strict;

no warnings;

use AI::Categorizer;

use AI::Categorizer::Collection::Files;

use Algorithm::NaiveBayes::Model::Frequency;

use File::Spec;

use File::Copy::Vigilant qw(copy move);

use File::ChangeNotify;

use File::Basename;

our $cat_file = 'E:\\FlowData\\PreProcess\\cats.txt';

our $path = 'E:\\FlowData\\Inbound';

my $watcher = File::ChangeNotify->instantiate_watcher(

directories => [$path],

filter => qr/\.(?:xml|txt)$/,

exclude => [ qr/(?:test|temp|audit|error)/is ],

);

use AI::Categorizer::Learner::NaiveBayes;

our %files;

our $nb = AI::Categorizer::Learner::NaiveBayes->restore_state('E:\\FlowData\\PreProcess\\state');

print "Begining to watch $path for changes.\n";

while ( my @events = $watcher->wait_for_events() ) {

sleep(5);

foreach my $event (@events) {

if ( $event->type() eq "create" ) {

sleep(2);

my ( $filename, $dirs, $suffix ) = fileparse( $event->path() );

print "Detected " . $event->type() . " on ".$filename."\n";

if ( $filename =~ m/\.txt$/ ) {

print "Moving assumed journal $filename\n";

move( $dirs . $filename, "E:\\FlowData\\Journal\\" . $filename, check => 'md5', retries => 'infinite' );

} else {

open my $out_fh, '>', $cat_file;

opendir( DIR, $path ) or die $!;

while ( my $file = readdir(DIR) ) {

# Use a regular expression to ignore files beginning with a period

next if ( $file =~ m/^\./ );

print {$out_fh} "$file \n";

}

closedir(DIR);

close $out_fh;

my $c = new AI::Categorizer::Collection::Files( path => $path, category_file => $cat_file );

while ( my $document = $c->next ) {

my $hypothesis = $nb->categorize($document);

print "Classified ", $hypothesis->document_name();

print " as ", $hypothesis->best_category, "\n";

my $original = $path . "\\" . $hypothesis->document_name();

my $classified = 'E:\\FlowData\\PreProcess\\' . $hypothesis->best_category() . '\\' . $hypothesis->document_name();

if ( $original =~ /^(.*)$/ ) {

$original = $1; # $data now untainted

} else {

die "Bad data in $original"; # log this somewhere

}

if ( $classified =~ /^(.*)$/ ) {

$classified = $1; # $data now untainted

} else {

die "Bad data in $classified"; # log this somewhere

}

move( $original, $classified, check => 'md5', retries => 'infinite' );

}

Classification at it’s simplest.

Data classification (simple & naive)

Recent Tweets

Unknown Feed