Training a ML Model to Solve a Classification Problem

Continuing the conversation form the J-shaped Pattern thread, I am including here the code for training a model for pattern recognition.

I hope I have annotated the code sufficiently to clarify the steps taken.

Charts:

Performance - tracks the cost for each iteration during the training of the model for Training and Validation sets. Training Cost shows how well the model is learning, while Validation Cost, tells us how well the model is generalizing. Usually you will see them diverging at some point.

ROC - stands for Receiver Operating Characteristic and indicates how well the classifier (model) is separating the classes.

// Pattern Recognition using Logistic Regression
// Aron Pipa, April 2019

function trace( x )
{
    _TRACE( MxToString( x ) ) ;
}
function rand( rows , cols )
{
    // matrix with random values
    x = Matrix( rows , cols )  ;

    for( i = 0 ; i < rows ; i ++ )
        for( j = 0; j < cols ; j ++ )
            x[i][j] = mtRandom();

    return x ;
}
function repmat( x, n )
{
    return x @ Matrix( 1, n , 1 ) ;
}
function cat( x, y )
{
    // concatenate horizontally
    rows = MxGetSize( x, 0 ) ;
    xcols = MxGetSize( x, 1 ) ;
    ycols = MxGetSize( y, 1 ) ;
    cols = xcols + ycols ;
    z = Matrix( rows , cols ) ;

    for( i = 0 ; i < rows; i ++ )
    {
        for( j = 0 ; j < cols; j ++ )
        {

            if( j >= xcols )
                value = y[i][j - xcols];
            else
                value = x[i][j];

            z[i][j] = value;
        }
    }

    return z ;
}
function extract ( fltr, data ) 
{
	// logical indexing
	m = MxGetSize ( data , 0 ) ; 
	n = MxSum ( fltr ) ; 
	mx = cat( MxTranspose( fltr ), MxTranspose( data) ) ;
    mx = MxSortRows( mx , False ) ;
    result = MxGetBlock( mx , 0, n - 1, 1, m ) ;
    return MxTranspose( result ) ;
}
function mean ( x ) 
{
	// calculates the mean horizonally 
	n = MxGetSize ( x, 1 )  ;
	return 1/n * x@Matrix ( n , 1, 1 ) ; 
}
function split ( x, t, ratio, x1, t1, x2, t2 ) 
{
	// split data in 2
	n = MxGetSize ( x, 1 ) ;
	m = MxGetSize ( x, 0 ) ; 
	
    div = ceil( ratio * n ) ;
    x1 = MxGetBlock( x, 0, m - 1 , 0, div - 1 );
    x2 = MxGetBlock( x, 0, m - 1 , div, n - 1 );

    t1 = MxGetBlock( t, 0, 0 , 0, div - 1 );
    t2 = MxGetBlock( t, 0, 0 , div, n - 1 );
    
}
function shuffle( x, t )
{
    //shuffle data
    m = MxGetSize( x, 0 ) ;
    n = MxGetSize( x , 1 ) ;

    xt = cat( MxTranspose( x ) , MxTranspose( t ) ) ;
    xt = cat( rand( n, 1 ), xt ) ;
    xt = MxSortRows( xt ) ;
    xt = MxGetBlock( xt , 0, n - 1, 1, m + 1 ) ;
    x  = MxGetBlock( xt , 0, n - 1, 0, m - 1 ) ;
    t  = MxGetBlock( xt , 0, n - 1, m, m ) ;
    x  = MxTranspose( x ) ;
    t  = MxTranspose( t ) ;
}
function logsig( x )
{
    return 1 / ( 1 + exp( 0 - x ) );
}
function loss( y , t )
{
	//cross-entropy loss
    return ( 0 - t ) * ln( y ) - ( 1 - t ) * ln( 1 - y );
}
function cost( y, t, w, lambda )
{
    n = MxGetSize( t, 1 ) ;
    reg =  lambda / ( 2 * n ) * MxSum( w ^ 2 ) ;
    return 1 / n * MxSum( loss( y , t ) ) + reg ;
}
function accuracy( y , t )
{
    n = MxGetSize( t , 1 );
    p = y > 0.5  ;
    return MxSum( p == t ) / n ;
}
function update( x, t, y, alpha, lambda , w, b  )
{
    n = MxGetSize( t, 1 ) ;
    reg = lambda / n * w;
    dz = ( y - t );
    dw = 1/n * dz @ MxTranspose( x ) + reg;
    db = 1/n * MxSum ( dz ) ;
    w -= alpha * dw;
    b -= alpha * db ; 
    
}
function predict( w, x, b )
{    
    return logsig( w@x + b  ) ;
}
function plotroc( y, t, chartname)
{
	// Receiver Operating Charachterstic 
	
    XYChartSetAxis( chartname , "[FPR]", "[TPR]", styleLine | styleThick );
    step = 0.1;
    for( th = 0 ; th < 1 ; th += step)
    {
        p = y > th;
        tp = mxsum( t == true  AND p == true ) ;
        tn = mxsum( t == false AND p == false ) ;

        fp = mxsum( p == true  AND t == false ) ;
        fn = mxsum( p == false AND t == true ) ;

        tpr = tp / ( tp + fn )  ;
        fpr = fp / ( fp + tn ) ;

        XYChartAddPoint( chartname, "", fpr, tpr, colorBrightGreen, colorbrightgreen ) ;
    }

    XYChartAddPoint( chartname, "", Null , Null ) ;

    for( th = 0 ; th < 1 + step; th += step )
        XYChartAddPoint( chartname, "", th, th, colorDarkGrey, colordarkgrey ) ;



}
function model( x, t, alpha, lambda, iterations )
{
    // Regularized Logistic Regresion 
    
    // shuffle data in dev set 
    shuffle( &x, &t ) ;
    
    // split dev set into training and validation sets
    split ( x, t, ratio = 0.7, &x_train, &t_train, &x_valid, &t_valid ); 

    m = MxGetSize( x, 0 ) ;
    n = MxGetSize( x, 1 ) ;
    

    //intialize parameters
    w = rand( 1 , m ) / sqrt( m ) ;
    b = mtRandom();

    XYChartSetAxis( "Perfomance", "[iteration]", "Costs: [red] Training, [green] Validation " ) ;

	// Gradient Descend 
    for( i = 0 ; i < iterations ; i ++ )
    {
        y_train = predict( w, x_train, b ) ;
        y_valid = predict( w, x_valid, b ) ;

        training_cost   = cost( y_train , t_train, w, lambda ) ;
        validation_cost = cost( y_valid , t_valid, w, lambda ) ;

        XYChartAddPoint( "Perfomance" , "", i , training_cost, colorRed, colorred ) ;
        XYChartAddPoint( "Perfomance" , "", i , validation_cost, colorgreen, colorgreen ) ;

        update( x_train, t_train, y_train, alpha, lambda, &w, &b );
    }

	_TRACEf( "%s %.2f%%", "training accuracy = ",   100 * accuracy( y_train, t_train )  ) ; 
	_TRACEf( "%s %.2f%%", "validation accuracy = ", 100 * accuracy( y_valid, t_valid )  ) ; 

	plotroc ( y_valid, t_valid, "Validation ROC" ) ; 
	
	// return learned parameters 
    VarSet ( "w", w ) ; 
    VarSet ( "b", b ) ; 
}

//define the target pattern using a vertical vector of features scaled 0 to 1
pattern = rand( features = 5 , 1 );

XYChartSetAxis ( "Patterns" , "feature num", "[green] target pattern, [red] mean of recognized patterns", styleLine|styledots )  ;
for( i = 0; i < features ; i ++ )
{
    XYChartAddPoint( "Patterns", "", i , pattern[i][0], colorGreen, colorgreen ) ;
}
XYChartAddPoint( "Patterns", "", Null, Null ) ; 


// setup gym: create a dataset made of copies of the target pattern and random sequences of the same length 
xtrue = repmat( pattern , n = 1000 ) ;    
xfalse = rand( features , n ) ;         

// concatenate, add labels, and shuffle: 
x = cat( xtrue, xfalse ) ;
t = cat( Matrix( 1, n , True ) , Matrix( 1, n , False ) ) ;
shuffle( &x, &t ) ;


// split data into dev and test sets  
split ( x, t, ratio = 0.7, &x_dev, &t_dev, &x_test, &t_test) ;

// train and return learned parameters 
model( x_dev, t_dev, alpha = 0.8, lambda = 0.5, iterations = 200 ) ;
//trace ( w ) ;
//trace ( b ) ; 

// use the learned parameters form the trained model to classify patterns in the test set 
y_test = predict( w, x_test, b ) ;   
plotroc ( y_test, t_test, "Test ROC" ) ; 

_TRACEf( "%s %.2f%%", "test accuracy = ", 100 * accuracy( y_test, t_test )  ) ; 

predictions = y_test > 0.5;
num = MxSum( predictions ) ;
if( num )
{
	// Extract recognised patterns form test set
	// calculate the mean aof their features 
	// visualise 
    patterns = extract ( predictions, x_test ) ;
    mu = mean ( patterns ) ; 

    for( i = 0; i < features ; i ++ )
    {
        XYChartAddPoint( "Patterns", "", i , mu[i][0], colorred , colorred) ;        
    }
}
30 Likes

@aron, I am eager to study your code.

I am already taking the course you recommended:

Thank you!

1 Like

Thank you very much for sharing this impressive piece of code!!!

I have had a first look and will dive deeper into it. I am a little bit stuck by several errors that I cannot fix. Could you please check if you have these errors on your configuration?

image

image

image

I also have a few preliminary questions:
The data are shuffled twice, once in data preparation phase (line 221) and reshuffled in the model function (line 163). Is it necessary?
In the same way, the data are split twice. It ends up with training the model on only 49% of the data. What is the reason why?

which AB version are you using ?

I am using 64bits version 6.30.0

This got nothing to do with AB versions. But with simple facts of AFL language.

For example,

MxGetSize of variable m of function extract is one based just like BarCount of 1-dim array is one based but bar index being zero based.

In similar fashion start/end rows and columns of MxGetBlock are zero based too! Last index can not be equal to MxGetSize but can only go as far as MxGetSize-1. So for example you get error in line 55 because it simple exceeds matrix range.

So change from

result = MxGetBlock( mx , 0, n-1, 1, m );

to

mx_col = MxGetSize( mx, 1)-1;
result = MxGetBlock( mx , 0, n-1, 1, Min(mx_col, m) );

I am not sure whether the 1 of start column is intended to be 1 or whether it is because of translation from one-based languages such as MatLab.

So it could be that you would have to change to

result = MxGetBlock( mx , 0, n-1, 0, Min(mx_col, m)  );

since AB indexing is zero based.

etc. for possible other fixes.

3 Likes

See this:

No its not, you may comment out the line calling shuffle() inside the model() function or better add an additional argument to control it.

We need 2 splits to the data to create training, validation, and test datasets, there is no reason other than this.
You can use a bigger ratio in split () function to get more data into training set

3 Likes

Installing last beta (6.30.5) solves the problem.
Thank you.

@Waz,

Once again and independently from used AB version you guys still make assumption that size of matrix data will always have same size as matrix mx of function extract (for example). Matrix data is still a different matrix than matrix mx.

Don't make assumptions on sizes. It doesn't matter whether it is 1-dimensional array or multi-dim array and it does not matter which version is used.
https://www.amibroker.com/kb/2014/09/22/do-not-make-assumptions-on-number-of-bars/

2 Likes

'data' is a m by n matrix
concatenating a vertical vector to transposed 'data' we get
'mx', a n by m + 1 matrix.
After sorting 'mx' rows by first column, this column is no longer needed.
The output matrix will use only the columns 1 to m ( including m) of 'mx'

@aron,

I am talking about the technical error as shown in picture by @Waz.

Once again.. it is assumed that size will fit always. It is clearly shown in picture that sizes are not always fitting and getting exceeded in e.g. line 55 of his picture. So made assumptions are incorrect.

The errors got nothing to do with version. But one can ensure error free run for any version.

And once again MxGetSize() is one-based but indexes go from zero to MxGetSize-1. Check extract() function. Also check n=MxSum ( fltr ); there which may become zero. So n-1 may become negative et voila you will get Error because of being outside range. That's what I am talking about.

For example from original function extract()...

function extract ( fltr, data ) 
{
	// by Aron P.
	// logical indexing
	m = MxGetSize ( data , 0 ) ; 
	n = MxSum ( fltr ) ; 
	mx = cat( MxTranspose( fltr ), MxTranspose( data) ) ;
	mx = MxSortRows( mx , False ) ;
	result = MxGetBlock( mx , 0, n - 1, 1, m ) ;
	return MxTranspose( result ) ;
}

... I modified that one to this one:

function extract(fltr, data) 
{
	// by Aron P.
	// fix to run error free by fxshrat
	// logical indexing
	if ( typeof(fltr) == "matrix" && typeof(data) == "matrix"  ) {  
		m = MxGetSize(data, 0); 
		n = MxSum ( fltr ); 
		mx = cat( MxTranspose(fltr), MxTranspose(data) );
		mx = MxSortRows( mx , False );
		
		mx_row = MxGetSize(mx, 0)-1;
		mx_col = MxGetSize(mx, 1)-1;
		
		end_row = Max(0, Min(mx_row, n-1));
		start_col = Min(1, mx_col);
		end_col = Max(start_col, Min(m, mx_col));   

		mx_block = MxGetBlock(mx, 0, end_row, start_col, end_col);
		result = MxTranspose(mx_block);
	} else {
		result = Matrix(1,1);
		_TRACE( "extract(): fltr and/or data are not type matrix!" );
		// or use
		//Error( "extract(): fltr and/or data are not type matrix!" );		
	}
    return result;
}

Here is complete "fixed one" that runs error free in Version 6.30.0 32-bit and 64-bit also but not just in 6.30.5.
Run formula editor's "Code check and profile" in 6.30.0. It will run error free.
I only "fixed" functions extract(), loss(), accuracy() and update(). Others may have to be checked also.

/// Pattern Recognition using Logistic Regression
/// Aron Pipa, April 2019
/// @link https://forum.amibroker.com/t/training-a-ml-model-to-solve-a-classification-problem/12719/1
/// fix of functions to run them error free on any occasion by fxshrat@gmail.com
/// @link https://forum.amibroker.com/t/training-a-ml-model-to-solve-a-classification-problem/12719/12

function trace( x )
{
    _TRACE( MxToString( x ) ) ;
}

function rand( rows , cols )
{
    // matrix with random values
    x = Matrix( rows , cols )  ;

    for( i = 0 ; i < rows ; i ++ )
        for( j = 0; j < cols ; j ++ )
            x[i][j] = mtRandom();

    return x ;
}

function repmat( x, n )
{
    return x @ Matrix( 1, n , 1 ) ;
}

function cat( x, y )
{
    // concatenate horizontally
    rows = MxGetSize( x, 0 );
    xcols = MxGetSize( x, 1 );
    ycols = MxGetSize( y, 1 );
    cols = xcols + ycols;
    z = Matrix( rows , cols );
    for( i = 0 ; i < rows; i ++ )
    {
        for( j = 0 ; j < cols; j ++ )
        {
            if( j >= xcols )
                value = y[i][j - xcols];
            else
                value = x[i][j];
            z[i][j] = value;
        }
    }
    return z;
}

function extract(fltr, data) 
{
	// by Aron P.
	// fix to run error free by fxshrat
	// logical indexing
	if ( typeof(fltr) == "matrix" && typeof(data) == "matrix"  ) {  
		m = MxGetSize(data, 0); 
		n = MxSum ( fltr ); 
		mx = cat( MxTranspose(fltr), MxTranspose(data) );
		mx = MxSortRows( mx , False );
		
		mx_row = MxGetSize(mx, 0)-1;
		mx_col = MxGetSize(mx, 1)-1;
		
		end_row = Max(0, Min(mx_row, n-1));
		start_col = Min(1, mx_col);
		end_col = Max(start_col, Min(m, mx_col));   

		mx_block = MxGetBlock(mx, 0, end_row, start_col, end_col);
		result = MxTranspose(mx_block);
	} else {
		result = Matrix(1,1);
		_TRACE( "extract(): fltr and/or data are not type matrix!" );
		// or use
		//Error( "extract(): fltr and/or data are not type matrix!" );		
	}
    return result;
}

function mean ( x ) 
{
	// calculates the mean horizonally 
	n = MxGetSize ( x, 1 )  ;
	return 1/n * x @ Matrix ( n, 1, 1 ); 
}

function split ( x, t, ratio, x1, t1, x2, t2 ) 
{
	// split data in 2
	n = MxGetSize ( x, 1 ) ;
	m = MxGetSize ( x, 0 ) ; 
	
    div = ceil( ratio * n );
    x1 = MxGetBlock( x, 0, m - 1, 0, div - 1 );
    x2 = MxGetBlock( x, 0, m - 1, div, n - 1 );

    t1 = MxGetBlock( t, 0, 0, 0, div - 1 );
    t2 = MxGetBlock( t, 0, 0, div, n - 1 );    
}

function shuffle( x, t )
{
    //shuffle data
    m = MxGetSize( x, 0 );
    n = MxGetSize( x, 1 );

    xt = cat( MxTranspose( x ), MxTranspose( t ) );
    xt = cat( rand( n, 1 ), xt );
    xt = MxSortRows( xt ) ;
    xt = MxGetBlock( xt, 0, n - 1, 1, m + 1 );
    x  = MxGetBlock( xt, 0, n - 1, 0, m - 1 );
    t  = MxGetBlock( xt, 0, n - 1, m, m );
    x  = MxTranspose( x );
    t  = MxTranspose( t );
}

function logsig( x )
{
    return 1 / ( 1 + exp( 0 - x ) );
}

function loss( y, t )
{
	// by Aron P.
	// fix to run error free by fxshrat
	if ( typeof(t) == "matrix" && typeof(y) == "matrix"  ) {  
		rows_y = MxGetSize(y, 0);
		cols_y = MxGetSize(y, 1);
		rows_t = MxGetSize(t, 0);
		cols_t = MxGetSize(t, 1);
		
		if( rows_y == rows_t && cols_y == cols_t ) {				
			//cross-entropy loss
			result = ( 0 - t ) * ln( y ) - ( 1 - t ) * ln( 1 - y );
		} else {	
			result = Matrix(1,1);
			_TRACE( "loss(): sizes of y and t are not equal!" );
			// or use
			//Error( "loss(): sizes of y and t are not equal!" );
		}		
	} else { 
		result = Matrix(1,1);
		_TRACE( "loss(): y and/or t are not type matrix!" );
		// or use
		//Error( "loss(): y and/or t are not type matrix!" );
	}
    return result;
}

function cost( y, t, w, lambda )
{
    n = MxGetSize( t, 1 ) ;
    reg =  lambda / ( 2 * n ) * MxSum( w ^ 2 ) ;
    result = 1 / n * MxSum( loss( y , t ) ) + reg;
    return result;
}

function accuracy( y , t )
{    
    // by Aron P.
	// fix to run error free by fxshrat
	if ( typeof(t) == "matrix" && typeof(y) == "matrix"  ) {    
		p = y > 0.5;
		n = MxGetSize( t, 1 );
		
		rows_p = MxGetSize(p, 0);
		cols_p = MxGetSize(p, 1);
		rows_t = MxGetSize(t, 0);
		cols_t = MxGetSize(t, 1);
		
		if( rows_p == rows_t && cols_p == cols_t ) {				
			result = MxSum( p == t ) / n; 
		} else {	
			result = 0;
			_TRACE( "accuracy(): sizes of y and t are not equal!" );
			// or use
			//Error( "accuracy(): sizes of y and t are not equal!"  );
		}		
    } else {
		result = 0; 
		_TRACE( "accuracy(): y and/or t are not type matrix!" );
		// or use
		//Error( "accuracy(): y and/or t are not type matrix!" );
	}   
    //
    return result;
}

function update( x, t, y, alpha, lambda, w, b  )
{
    // by Aron P.
	// fix to run error free by fxshrat    
    if( typeof(y) == "matrix" AND typeof(t) == "matrix" ) {    
		n = MxGetSize( t, 1 );
		reg = lambda / n * w;
		
		rows_y = MxGetSize(y, 0);
		cols_y = MxGetSize(y, 1);
		rows_t = MxGetSize(t, 0);
		cols_t = MxGetSize(t, 1);
		
		if( rows_y == rows_t && cols_y == cols_t ) {
			dz = ( y - t );
			dw = 1/n * dz @ MxTranspose( x ) + reg;
			db = 1/n * MxSum ( dz ) ;
			w -= alpha * dw;
			b -= alpha * db; 
		} else {	
			w = b = Matrix(1,1);
			_TRACE( "update(): sizes of y and t are not equal!" );
			// or use
			//Error( "update(): sizes are not equal!" );
		}
	} else {
		w = b = Matrix(1,1);
		_TRACE( "update(): y and/or t are not type matrix!" );
		// or use
		//Error( "update(): y an/or t are not type matrix!" );
	}   
    
}

function predict( w, x, b )
{    
    return logsig( w@x + b  );
}

function plotroc( y, t, chartname)
{
	// Receiver Operating Charachterstic 	
    XYChartSetAxis( chartname , "[FPR]", "[TPR]", styleLine | styleThick );
    step = 0.1;
    for( th = 0 ; th < 1 ; th += step)
    {
        p = y > th;
        tp = mxsum( t == true  AND p == true ) ;
        tn = mxsum( t == false AND p == false ) ;

        fp = mxsum( p == true  AND t == false ) ;
        fn = mxsum( p == false AND t == true );

        tpr = tp / ( tp + fn )  ;
        fpr = fp / ( fp + tn ) ;

        XYChartAddPoint( chartname, "", fpr, tpr, colorBrightGreen, colorbrightgreen );
    }

    XYChartAddPoint( chartname, "", Null , Null );

    for( th = 0 ; th < 1 + step; th += step )
        XYChartAddPoint( chartname, "", th, th, colorDarkGrey, colordarkgrey ) ;
}

function model( x, t, alpha, lambda, iterations )
{
    // Regularized Logistic Regresion 
    
    // shuffle data in dev set 
    shuffle( &x, &t ) ;
    
    // split dev set into training and validation sets
    split ( x, t, ratio = 0.7, &x_train, &t_train, &x_valid, &t_valid ); 

    m = MxGetSize( x, 0 ) ;
    n = MxGetSize( x, 1 ) ;
    

    //intialize parameters
    w = rand( 1 , m ) / sqrt( m ) ;
    b = mtRandom();

    XYChartSetAxis( "Perfomance", "[iteration]", "Costs: [red] Training, [green] Validation " ) ;

	// Gradient Descend 
    for( i = 0 ; i < iterations ; i ++ )
    {
        y_train = predict( w, x_train, b ) ;
        y_valid = predict( w, x_valid, b ) ;

        training_cost   = cost( y_train , t_train, w, lambda ) ;
        validation_cost = cost( y_valid , t_valid, w, lambda ) ;

        XYChartAddPoint( "Perfomance" , "", i , training_cost, colorRed, colorred ) ;
        XYChartAddPoint( "Perfomance" , "", i , validation_cost, colorgreen, colorgreen ) ;

        update( x_train, t_train, y_train, alpha, lambda, &w, &b );
    }

	_TRACEf( "%s %.2f%%", "training accuracy = ",   100 * accuracy( y_train, t_train )  ) ; 
	_TRACEf( "%s %.2f%%", "validation accuracy = ", 100 * accuracy( y_valid, t_valid )  ) ; 

	plotroc ( y_valid, t_valid, "Validation ROC" ) ; 
	
	// return learned parameters 
    VarSet ( "w", w ) ; 
    VarSet ( "b", b ) ; 
}

//define the target pattern using a vertical vector of features scaled 0 to 1
pattern = rand( features = 5 , 1 );

XYChartSetAxis ( "Patterns" , "feature num", "[green] target pattern, [red] mean of recognized patterns", styleLine|styledots )  ;
for( i = 0; i < features ; i ++ )
{
    XYChartAddPoint( "Patterns", "", i , pattern[i][0], colorGreen, colorgreen ) ;
}
XYChartAddPoint( "Patterns", "", Null, Null ) ; 


// setup gym: create a dataset made of copies of the target pattern and random sequences of the same length 
xtrue = repmat( pattern , n = 1000 ) ;    
xfalse = rand( features , n ) ;         

// concatenate, add labels, and shuffle: 
x = cat( xtrue, xfalse ) ;
t = cat( Matrix( 1, n , True ) , Matrix( 1, n , False ) ) ;
shuffle( &x, &t ) ;


// split data into dev and test sets  
split ( x, t, ratio = 0.7, &x_dev, &t_dev, &x_test, &t_test) ;

// train and return learned parameters 
model( x_dev, t_dev, alpha = 0.8, lambda = 0.5, iterations = 200 ) ;
//trace ( w ) ;
//trace ( b ) ; 

// use the learned parameters form the trained model to classify patterns in the test set 
y_test = predict( w, x_test, b ) ;   
plotroc ( y_test, t_test, "Test ROC" ) ; 

_TRACEf( "%s %.2f%%", "test accuracy = ", 100 * accuracy( y_test, t_test )  ) ; 

predictions = y_test > 0.5;
num = MxSum( predictions ) ;
if( num )
{
	// Extract recognised patterns form test set
	// calculate the mean aof their features 
	// visualise 
    patterns = extract(predictions, x_test);
    mu = mean(patterns); 

    for( i = 0; i < features; i++ )
    {
        XYChartAddPoint( "Patterns", "", i , mu[i][0], colorred , colorred) ;        
    }
}

4 Likes

The errors on the screenshot are because of problem with math functions not because of extract() function which is called after making sure that 'n' is not zero.
Extract() should not output any matrix if n == 0.

I mean nothing valid , it can output null , throw and error or something.
One can encapsulate everything outside the functions into another function so that it can be fail safe

get_me_those_damn_parameters ( pattern ) 
{
    all the procedure here 
    return parameters 
}

but this is not the scope of this thread, I tried to keep everything as simple as possible so the code can rum and others can understand what is going on.
I do not want to spend time discussing here how to perfect the functions included here so that they can be used in every kind of situation - this could take place in another thread

"The errors on the screenshot are because of problem with math functions not because of extract() function which is called after making sure that
'n' is not zero."

One of errors in extract come from MxGetBlock indexes getting exceeded which is result of that. The images are pretty clear. My "fix" was just example how to prevent exceeding range and error.

"I mean nothing valid , it can output null , throw and error or something."

Sure, that's my point. Code should take care for such cases.

My philosophy is that code should run issue free on any occasion.

"I tried to keep everything as simple as possible".

And no one did blame you. I was basically responding to @Waz.

1 Like

Hi @aron,
thanks for share your ML system,
Im learning your code, word by word. When we explore good result like below attacted picture , how to get and put that data into model.
Thanks you again!
Albert,
Result%20data%20train

Hi @Albert88
model( x, t, alpha, lambda, iterations ) trains the model to recognize a certain pattern using the training dataset, and will output weights and and bias w and b.

In order to recognize similar pattern on a new data set, you use predict( w, x, b ) that will output a score ( 0 to 1 )

the following code creates the feature matrix that you can use as input x

bi = BarIndex();
len = 5 ;  // pattern length 
fltr = bi>= len ; 
hh = IIf ( fltr, HHV( close, len ), Null ); 
ll = IIf ( fltr, llv( close, len ), Null );

x = Matrix( len, BarCount  ) ;
nulls = 0; 
for( i = 0 ; i < len ; i ++ )
{
    var = Ref( Close, - i ) ;
    ftr = ( var - ll ) / ( hh - ll ) ;
    x = MxSetBlock( x , i, i , 0, BarCount - 1, ftr ) ;
    AddColumn( ftr , StrFormat ("x%.f",i+ 1)  );
    nulls = Max( nulls, NullCount( ftr ) ) ;
}

Filter = bi >= nulls;

// remove nulls form matrix 
m = MxGetSize ( x, 0 ) ; 
n = MxGetSize ( x, 1 ) ; 
x = MxGetBlock ( x, 0, m -1 , Nulls , n -1 ) ; 
_TRACE( MxToString( ( x ) ) ) ;
6 Likes

Hi @aron,
Your above code help me a lots.
When output we trained is good, with high accuracy, how to put or update it(w and b) into model because everytime i explored(analysis), there are some different outputs.

Here is a more complete example:

1) Select and save the price pattern

image

2) Run Exploration

image

3) Visualize on chart
The highlighted bars and arrows mark the last bar in the detected pattern

image

// Training a logistic regression model to recognize price patterns 
// Aron Pipa, November 2019

function trace( x )
{   // just a warpper for the _trace() function
    _TRACE( MxToString( x ) ) ;
}
function rand( rows , cols )
{    // returns a matrix of random values (0 to 1 ) 

    x = Matrix( rows , cols )  ;

    for( i = 0 ; i < rows ; i ++ )
        for( j = 0; j < cols ; j ++ )
            x[i][j] = mtRandom();

    return x ;
}
function repmat( x, n )
{   // returns a matrix repeating the vector 'x' n-times
    return x @ Matrix( 1, n , 1 ) ;
}
function cat( x, y )
{   // concatenates horizontally

    rows = MxGetSize( x, 0 ) ;
    xcols = MxGetSize( x, 1 ) ;
    ycols = MxGetSize( y, 1 ) ;
    cols = xcols + ycols ;
    z = Matrix( rows , cols ) ;

    for( i = 0 ; i < rows; i ++ )
    {
        for( j = 0 ; j < cols; j ++ )
        {

            if( j >= xcols )
                value = y[i][j - xcols];
            else
                value = x[i][j];

            z[i][j] = value;
        }
    }

    return z ;
}
function extract( fltr, data )
{   // performs logical indexing based on 'fltr' and extracts blocks from input matrix' data'

    m = MxGetSize( data , 0 ) ;
    n = MxSum( fltr ) ;
    mx = cat( MxTranspose( fltr ), MxTranspose( data ) ) ;
    mx = MxSortRows( mx , False ) ;
    result = MxGetBlock( mx , 0, n - 1, 1, m ) ;
    return MxTranspose( result ) ;
}
function mean( x )
{   // calculates the mean horizonally

    n = MxGetSize( x, 1 )  ;
    return 1 / n * x@Matrix( n , 1, 1 ) ;
}
function split( x, t, ratio, x1, t1, x2, t2 )
{   // splits 'x'and 't' in 2 based on input ratio

    n = MxGetSize( x, 1 ) ;
    m = MxGetSize( x, 0 ) ;

    div = ceil( ratio * n ) ;
    x1 = MxGetBlock( x, 0, m - 1 , 0, div - 1 );
    x2 = MxGetBlock( x, 0, m - 1 , div, n - 1 );

    t1 = MxGetBlock( t, 0, 0 , 0, div - 1 );
    t2 = MxGetBlock( t, 0, 0 , div, n - 1 );

}
function shuffle( x, t )
{  //shuffles input feature matrix 'x' and targets 't'

    m = MxGetSize( x, 0 ) ;
    n = MxGetSize( x , 1 ) ;

    xt = cat( MxTranspose( x ) , MxTranspose( t ) ) ;
    xt = cat( rand( n, 1 ), xt ) ;
    xt = MxSortRows( xt ) ;
    xt = MxGetBlock( xt , 0, n - 1, 1, m + 1 ) ;
    x  = MxGetBlock( xt , 0, n - 1, 0, m - 1 ) ;
    t  = MxGetBlock( xt , 0, n - 1, m, m ) ;
    x  = MxTranspose( x ) ;
    t  = MxTranspose( t ) ;
}
function logsig( x )
{   //sigmoid function
    return 1 / ( 1 + exp( 0 - x ) );
}
function loss( y , t )
{  //cross-entropy loss function

    return ( 0 - t ) * log( y ) - ( 1 - t ) * log( 1 - y );
}
function cost( y, t, w, lambda )
{   //calculates the cost as mean of loss 
    n = MxGetSize( t, 1 ) ;
    reg =  lambda / ( 2 * n ) * MxSum( w ^ 2 ) ;
    return 1 / n * MxSum( loss( y , t ) ) + reg ;
}
function accuracy( y , t )
{   // calculates accuracy in classification of output 'y as target 't' 
    n = MxGetSize( t , 1 );
    p = y > 0.5  ;
    return MxSum( p == t ) / n ;
}
function update( x, t, y, alpha, lambda , w, b )
{   // update procedure for weights and bias during gradient descent

    n = MxGetSize( t, 1 ) ;
    reg = lambda / n * w;
    dz = ( y - t );
    dw = 1 / n * dz @ MxTranspose( x ) + reg;
    db = 1 / n * MxSum( dz ) ;
    w -= alpha * dw;
    b -= alpha * db ;

}
function classify( w, x, b )
{   // performs classification based on input feature matrix 'x' and parameters 'w' and 'b'
    return logsig( w@x + b ) ;
}
function train( x, t, alpha, lambda, iterations )
{  // trains a logistic regression model

    global id; 
    // shuffle data in dev set
    shuffle( &x, &t ) ;

    // split dev set into training and validation sets
    split( x, t, ratio = 0.7, &x_train, &t_train, &x_valid, &t_valid );

    m = MxGetSize( x, 0 ) ;
    n = MxGetSize( x, 1 ) ;


    //intialize parameters
    w = rand( 1 , m ) / sqrt( m ) ;
    b = mtRandom();

    XYChartSetAxis( "Perfomance", "[iteration]", "Costs: [red] Training, [green] Validation " ) ;

    // Training procedure using Gradient Descend
    for( i = 0 ; i < iterations ; i ++ )
    {
        y_train = classify( w, x_train, b ) ;
        y_valid = classify( w, x_valid, b ) ;

        training_cost   = cost( y_train , t_train, w, lambda ) ;
        validation_cost = cost( y_valid , t_valid, w, lambda ) ;

        XYChartAddPoint( "Perfomance" , "", i , training_cost, colorRed, colorred ) ;
        XYChartAddPoint( "Perfomance" , "", i , validation_cost, colorgreen, colorgreen ) ;

        update( x_train, t_train, y_train, alpha, lambda, &w, &b );
    }

    _TRACEf( "%s %.2f%%", "training accuracy = ",   100 * accuracy( y_train, t_train ) ) ;
    _TRACEf( "%s %.2f%%", "validation accuracy = ", 100 * accuracy( y_valid, t_valid ) ) ;

   // plotroc( y_valid, t_valid, "Validation ROC" ) ;

    // return learned parameters
    staticVarSet( id + "w", w ) ;
    staticvarSet( id + "b", b ) ;
}
function defpat( var )
{   //defines the target pattern by selecting a price sequence on chart 

    bi = BarIndex();
    fb = BeginValue( bi );
    lb = EndValue( bi ) ;

    len = lb - fb + 1;
    hh = HHV( var , len ) ;
    ll = LLV( var , len ) ;

    result = 0 ;

    if( len )
    {
        x = Matrix( len , 1 ) ;
        row = 0;

        for( i = fb ; i<lb + 1; i ++)
        {
            x[row][0] = ( var[i] - ll [lb] ) / ( hh [lb] - ll [lb] );
            row ++;
        }

        result = x ;
    }

    return result ;
}

//global variables ; 
id = "pr"; 
ticker = Name();
save  = ParamTrigger ( "save" , "save pattern" ) ;  
clear = ParamTrigger ( "clear", "clear all" ) ; 

if ( clear )
StaticVarRemove ( id + "*" ) ; 


//Training Parameters 
trdsize = 500;    // training dataset size (total number of samples)
alpha = 0.5;      // learning rate
lambda = 0.4;     // regularization parameter
iterations = 100; // iterations for the gradient descent 
cutoff = 0.5;  	  // score values above cutoff score will be classified as 'true' and below as 'false';



if( Status( "action" ) == actionIndicator )
{
	//define the target pattern using a column vector of features scaled 0 to 1
    x = defpat( close ) ;

    if( save )
    {
        StaticVarSet( id  +  "pattern", x ) ;
        StaticVarSettext( id + "symbol" , ticker ) ;
    }

    barcolor = StaticVarGet( id + ticker + "barcolor" ) ;
    shape    = StaticVarGet( id + ticker + "shape" );

	clr = IIf ( barcolor ,barcolor , colorGrey50 ); 
    Plot( C, "", clr , styleBar );
    PlotShapes( shape , colorGreen , 0, Low ) ;

    pattern = StaticVarGet( id + "pattern" ) ;
    Title = "Select and save a pattern" ;

    if( pattern )
        Title = strformat ( "Saved Pattern = %s\n%s" , StaticVarGet (id + "symbol" ) , MxToString( pattern ) )  ;

}

else
{
    pattern = StaticVarGet(id +  "pattern" );

    if( pattern )
    {
        features = MxGetSize( pattern, 0 ) ;

        if( Status( "stocknum" ) == 0 ) // training procedure 
        {

            XYChartSetAxis( "Patterns" , "feature num", "[green] target pattern, [red] mean of recognized patterns", styleLine | styledots )  ;

            for( i = 0; i < features ; i ++ )
            {
                XYChartAddPoint( "Patterns", "", i , pattern[i][0], colorGreen, colorgreen ) ;
            }

            XYChartAddPoint( "Patterns", "", Null, Null ) ;


            // setup gym: create a dataset made of copies of the target pattern and random sequences of the same length
            xtrue = repmat( pattern , n = trdsize/2 ) ;
            xfalse = rand( features , n ) ;

            // concatenate, add labels, and shuffle:
            x = cat( xtrue, xfalse ) ;
            t = cat( Matrix( 1, n , True ) , Matrix( 1, n , False ) ) ;
            shuffle( &x, &t ) ;


            // split data into dev and test sets
            split( x, t, ratio = 0.7, &x_dev, &t_dev, &x_test, &t_test ) ;

            // train and return learned parameters
            train( x_dev, t_dev, alpha, lambda , iterations ) ;

            // retreive the parameters learved form training and classify patterns in the test set
            w = StaticVarGet( id + "w" ) ;
            b = StaticVarGet( id + "b" ) ;
            y_test = classify( w, x_test, b ) ;

            _TRACEf( "%s %.2f%%", "test accuracy = ", 100 * accuracy( y_test, t_test ) ) ;

            predictions = y_test > cutoff;
            num = MxSum( predictions ) ;

            if( num )
            {
                // Extract recognised patterns form test set
                // calculate the mean of their features
                // visualise
                patterns = extract( predictions, x_test ) ;
                mu = mean( patterns ) ;

                for( i = 0; i < features ; i ++ )
                {
                    XYChartAddPoint( "Patterns", "", i , mu[i][0], colorred , colorred ) ;
                }
            }


        }

        // prepare input feature matrix to classify patterns on the real database
        x = Matrix( features, BarCount ) ;
        bi = BarIndex(); 

        hh = HHV( close , features ) ;
        ll = LLV( close , features ) ;
        hh = IIf ( bi>=features, hh, Null ); 
        ll = IIf ( bi>=features, ll, Null ); 

        nulls = 0 ;
        row = features -1; 
        for( i = 0 ;i < features; i ++)
        {
            var = Ref( close , - i) ;
            pred = ( var - ll ) / ( hh - ll ); 
            x = MxSetBlock( x, row, row , 0, BarCount - 1 , pred  );
            nulls = Max( nulls , NullCount( var) ) ;
            row --; 
        }

        // retreive the learned paramters 
        w = StaticVarGet( id + "w" ) ;
        b = StaticVarGet( id + "b" ) ;  
        
        // perform classification       
        score = classify ( w, x, b ) ;        
        score = MxGetBlock ( score , 0, 0 , 0 , BarCount-1 ,  True ) ; 
        
        barcolor = IIf ( score > cutoff , colorYellow , colorGrey50 ) ; 
        shape    = IIf ( score > cutoff , shapeUpArrow , shapenone ) ; 
        
        StaticVarSet ( id + ticker + "barcolor", barcolor ) ; 
        StaticVarSet ( id + ticker + "shape", shape ) ; 

        Filter = score > cutoff; 
        AddColumn ( score , "Score" ) ; 
        SetSortColumns ( -3 ) ; 
        
    }
}
13 Likes

Hello @aron

May I ask you, besides classical patterns recognition (eg triabgles, Head&Shoulders), where else could you apply Machine Learning?

For example I would like to detect if a break of price through a resistance is going to succeed and move higher up, or it is going to fail and return bellow the resistance again. Is it realistic to make a ML algo that will examine the conditions prior to the breakout and determine the outcome?

Hi Bob,
You can use something like the example below to produce a dataset defining features and labels and apply any classification ML algo.

// define breakout 
hh = HHV( High , pds = 5 ) ;
resistance = Ref( hh, - 1 ) ;
breakout = Cross ( Close , resistance )  ;



// define labels:
// True if price stays above the resistance for the next 3 bars after breakout, False otherwise
lookForward = 4  ;   
success = Null;

for( i = pds+ 1 ; i < BarCount - lookforward ; i ++ )
{
    if( breakout [i] )
    {
        success[i] = True ;

        for( j = i ; j < i + lookforward ; j ++ )
        {
            if( Close [j] < resistance[i] )
            {
                success[i] = False;
                break ;
            }
        }

    }
}


// define the features: 
// Close value for past 10 bars ( including current )  

lookback = 10 ;

for( i = 0 ; i < lookback ; i ++ )
{
    predictor = Ref( Close, -i ) ;
    AddColumn( predictor, "x" + i ) ;
}

Filter = !IsNull( success ) AND BarIndex() >= lookback ;
AddColumn( success , "target" );
3 Likes