www.pudn.com > NeuralNetworkSourceCode.zip > ALOPEX.CPP, change:2001-02-16,size:19104b
/****************************************************************************
* *
* ALOPEX *
* *
*****************************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <conio.h>
#include <math.h>
// FUNCTION PROTOTYPES
void ShowResults(int, long int,double temp, double alpha, double eta);
// DEFINES
#define MAXLAYERS 4 // MAX NUMBER OF LAYERS IN NET (IN OUT & HIDDEN)
#define MAXNEURONS 30 // MAX NUMBER OF NEURONS PER LAYER
#define MAXPATTERNS 20 // MAX NUMBER OF PATTERNS IN A TRAINING SET
#define MAXPREV 3 // NUMBER OF PREV EPOCH FOR WHICH WEIGHTS ARE MAINTAINED
#define ARCGRAN 250 // ARCHIVE GRANULARITY
#define NONE 0 // DO NOT ARCHIVE ERROR DATA
#define ALL 1 // ARCHIVE ALL ERROR DATA FOR EACH ITERATION
#define AVERAGE 2 // ARCHIVE AVERAGE ERROR DATA FOR EACH EPOCH
#define TRUE 1
#define FALSE 0
// ------------------------------------------------------------------------
class ALOPEX
{
private:
double W[MAXLAYERS][MAXNEURONS][MAXNEURONS]; // WEIGHTS MATRIX
double Wprev1[MAXLAYERS][MAXNEURONS][MAXNEURONS]; // previous WEIGHTS for correlation
double Neuron[MAXLAYERS][MAXNEURONS]; // Output of all net neurons
double ERROR[MAXPATTERNS][MAXNEURONS]; // Stored error over epoch/outlayer
double AvgErr;
// Alopex specific data
double delta; // The Alopex step size
double T; // Temperature for Alopex probability fn
double Ep1; // Sum error over iteration n-1
int AnnealCount;
double SumDELTAc;
// Topology
int NumLayers; // Number of layers
int OutLayerIndx; // array index of last layer
int LayerSize[MAXLAYERS]; // Number of neurons in the each layer
int NumWeights; // Tot number of weights
// Pattern data
double InPattern[MAXPATTERNS][MAXNEURONS]; // Input values for each pattern
double Desired[MAXPATTERNS][MAXNEURONS]; // desired value for each
// pattern/output
unsigned long int CurrIter; // Current iterations number.
long int Epoch; // Counts number of epochs
int EpochFlag; // True at end of each epoch
int NumPatterns; // Total patterns in training set
int CurrPat; // Current pattern used in training
int ConvCount; // The number of consecutive
// patterns within tolerance
int ConvergeFlg; // Flag indicates convergance has
// occurred
// PARMS
double Qzero; // For Neuron activation function
double ERRTOL; // min Error tolerance required for convergence
double ETA; // ALOPEX Version of learning rate
unsigned long int MAXITER; // Max iterations to do before stopping
// PRIVATE METHODS
double ErrFunc(double Target, double Actual); // Calc the log error
double CalcSumErr(); // Calc tot err over all patterns/outlayer neurons
void RecordErrors(); // At each iter, record error results
void ArchivePut(); // Keep a record on disk of net progress
public:
ALOPEX(void);
void GetTrnSet(char *Fname); // Load training set from file
void NextPat();
void GetInputs(void); // Load input layer w/ curr pattern
void RunNet(void); // Run the networks forward pass
void GetParms(char *); // Load parameters from parm file
double Sigmoid(double Net, double Tempr); // The non-linear squashing function
void SetRandomWeights(void); // Init weights to random values
int IsConverged(void); // Check for convergence
void AdaptWeights(void); // Modify weights by ALOPEX method
void SaveWeights(char *); // Save trained weights to file
int train(char *, char *); // Top level control of training the net
long int QueryEpoch(){return Epoch;}
double QueryQzero(){return Qzero;}
};
// MISC FLAGS
int ArchOn = AVERAGE; // Set to 1= ALL, NONE, AVERAGE or WORST to control
// data sent to ARCHIVE file for graphical
// analysis
FILE *ARCHIVE; // Archive Training sequence
// ------------------------------------------------------------------------
// METHOD DEFINITIONS
ALOPEX::ALOPEX() {
AvgErr = 0.0;
CurrIter=0;
Epoch=0;
AnnealCount=0;
CurrPat = 0; // Current pattern used in training
ConvCount=0;
ConvergeFlg=FALSE; // Flag indicates convergance
SumDELTAc=0.0;
ETA=2.0;
EpochFlag=FALSE;
}
/**************************************************************************
* Function GetTrnSet *
* Load training set *
**************************************************************************/
void ALOPEX::GetTrnSet(char *Fname) { // for small training sets
FILE *PFILE;
int x;
int pat,i,j;
double inVal;
PFILE = fopen(Fname,"r"); // batch
if (PFILE==NULL){
printf("\nUnable to open file \n");
exit(0);
}
fscanf(PFILE,"%d",&NumPatterns);
for (pat=0; pat<NumPatterns; pat++) {
for (i=0; i<LayerSize[0]; i++) {
fscanf(PFILE,"%lf",&inVal);
InPattern[pat][i]=inVal;
} /* endfor */
// Now get desired / expected values
for (i=0; i<LayerSize[OutLayerIndx]; i++) {
fscanf(PFILE,"%lf",&inVal);
Desired[pat][i]=inVal;
} /* endfor */
} /* endfor */
fclose(PFILE);
}
/******************************************************************************
* *
* Function GetParms *
* Loads topology from file spec'd file if avail. *
* otherwise tries DEFAULT.PRM *
* *
******************************************************************************/
void ALOPEX::GetParms(char *PrmFileName){
FILE *PRMFILE;
PRMFILE = fopen(PrmFileName,"r"); // batch
if (PRMFILE==NULL){
printf("\nUnable to open Parameter file: %s \n",PrmFileName);
printf("\nAttempting to open default file: PARMS1");
PRMFILE = fopen("PARMS1","r");
if (PRMFILE==NULL){
printf("\nUnable to open Parameter file\n");
exit(0);
}
}
printf("\nLoading Parameters from file: %s\n",PrmFileName);
fscanf(PRMFILE,"%lf",&Qzero);
fscanf(PRMFILE,"%lf",&ETA);
fscanf(PRMFILE,"%lf",&delta);
fscanf(PRMFILE,"%lf",&T);
fscanf(PRMFILE,"%ld",&MAXITER);
fscanf(PRMFILE,"%lf",&ERRTOL);
fscanf(PRMFILE,"%d",&NumLayers);
printf("&Qzero=%lf",Qzero);
printf("&delta=%lf",delta);
printf("T=%lf",T);
printf("MAXITER=%ld",MAXITER);
printf("ERRTOL=%lf",ERRTOL);
printf("\nNumber of layers=%d\n",NumLayers);
for (int i=0; i<NumLayers; i++) {
fscanf(PRMFILE,"%d",&LayerSize[i]);
printf("Number of neurons in layer %d = %d\n",i,LayerSize[i]);
}
OutLayerIndx = NumLayers-1; // accomodate 0 org'd arrays
fclose(PRMFILE);
NumWeights=0;
for (int lyr=OutLayerIndx; lyr>0; lyr--) //Visit each layer
NumWeights += LayerSize[lyr] * LayerSize[lyr-1];
printf("\n%d\n",NumWeights);
}
void ALOPEX::SetRandomWeights(){
int i,j,k;
double zWT;
double x;
//srand(6);
srand(5);
for (i=1; i<NumLayers; i++) {
for (k=0; k<LayerSize[i]; k++) {
for (j=0; j<=LayerSize[i-1]; j++) { // One extra for bias neuron
zWT=(double)rand();
zWT=zWT/16.0;
x=(double)rand();
x=x/32767;
if (x>0.5) {
zWT=-zWT;
} else {
} /* endif */
W[i][j][k] =zWT/32767.0;
printf("\nW(%d,%d,%d)%lf\n",i,j,k,W[i][j][k]);
}
}
}
}
void ALOPEX::NextPat(){
EpochFlag=FALSE;
CurrIter++;
CurrPat++; // Update the current pattern
if (CurrPat>=NumPatterns){
EpochFlag=TRUE;
CurrPat=0;
Epoch++;
}
}
/******************************************************************************
* FUNCTION GetInputs *
* *
* Loads input layer neurons with current pattern *
******************************************************************************/
void ALOPEX::GetInputs(){
int i,j,k;
for (i=0; i<LayerSize[0]; i++) {
Neuron[0][i]=InPattern[CurrPat][i]; // Show it to the neurons
}
}
/******************************************************************************
* FUNCTION RunNet *
* *
* Calculates outputs for all network neurons *
******************************************************************************/
void ALOPEX::RunNet(){
int lyr; // layer to calculate
int dNeuron; // dest layer neuron
int sNeuron; // src layer neuron
double SumNet;
double out;
for (lyr=1; lyr<NumLayers; lyr++) {
Neuron[lyr-1][LayerSize[lyr-1]]=1.0; //force bias neuron output to 1.0
for (dNeuron=0; dNeuron<LayerSize[lyr]; dNeuron++) {
SumNet=0.0;
for (sNeuron=0; sNeuron <= LayerSize[lyr-1]; sNeuron++) { //add 1 for bias
SumNet += Neuron[lyr-1][sNeuron] * W[lyr][sNeuron][dNeuron];
}
out=Sigmoid(SumNet,Qzero);
Neuron[lyr][dNeuron] = out;
}
}
}
/******************************************************************************
******************************************************************************/
void ALOPEX::RecordErrors(){
int j;
double x;
for (j=0; j<LayerSize[OutLayerIndx]; j++) {
ERROR[CurrPat][j] = ErrFunc(Desired[CurrPat][j], Neuron[OutLayerIndx][j]);
if ((ARCGRAN*(Epoch/ARCGRAN)) == Epoch) {
x=ERROR[CurrPat][j];
printf("Pattern: %d neuron[%d]=%lf / %lf ERROR= %lf\n",
CurrPat,j, Neuron[OutLayerIndx][j],Desired[CurrPat][j],x);
}
} /* endfor */
}
/******************************************************************************
* IsConverged *
* Calculates error for each neuron and each pattern *
******************************************************************************/
int ALOPEX::IsConverged(){
double dNETj;
double SumErr; //Cumulative error
int i, j;
int LocalConvFlg=TRUE;
SumErr=CalcSumErr();
if (SumErr>ERRTOL) {
return FALSE;
} else {
return TRUE;
} /* endif */
}
/******************************************************************************
******************************************************************************/
void ALOPEX::ArchivePut(){
int i, j;
//printf("\n");
AvgErr =CalcSumErr()/NumPatterns;
if (ArchOn==AVERAGE) {
if ((ARCGRAN*(Epoch/ARCGRAN)) == Epoch) { // only save ARCGRANth epochs
fprintf(ARCHIVE,"%ld %lf\n",Epoch,AvgErr );
printf("\nEpoch:%d ",Epoch);
printf("Avg=%lf\n",AvgErr);
} /* endif */
}
}
/******************************************************************************
******************************************************************************/
double ALOPEX::ErrFunc(double Target, double Actual) {
double E;
//------------------------------------------------------------
// Uncomment this section to use log error function
//if (Target>=.99) {
// E=log(1/Actual);
// }
// else {
// E=log(1/(1-Actual));
// } /* endif */
//------------------------------------------------------------
// Absolute error function
E=sqrt((Target-Actual)*(Target-Actual));
//------------------------------------------------------------
// Uncomment this section to use square error function
//E=(Target-Actual)*(Target-Actual);
//------------------------------------------------------------
return E;
}
/******************************************************************************
******************************************************************************/
double ALOPEX::CalcSumErr(){
int j,Pat;
double E,desire;
E=0.0;
for (Pat=0;Pat<NumPatterns ; Pat++) {
for (j=0; j<LayerSize[OutLayerIndx]; j++) {
E += ERROR[Pat][j];
}
} /* endfor */
return E;
}
void ALOPEX::AdaptWeights(){
double E,R,P, DELTAc;
double DELTAWEIGHT, DELTAERROR;
int lyr, i, j,k;
E=CalcSumErr();
DELTAERROR=E-Ep1;
Ep1=E; //Prev iter errors
for (lyr=OutLayerIndx; lyr>0; lyr--) { // Visit each layer
for (j=0; j<LayerSize[lyr]; j++) { // Visit each neuron
for (k=0; k<=LayerSize[lyr-1];k++ ) { // Visit all Weights (incl. bias)
DELTAWEIGHT= W[lyr][j][k]-Wprev1[lyr][j][k];
DELTAc= DELTAWEIGHT*DELTAERROR; //Calculate the correlation
SumDELTAc+=fabs(DELTAc); //Running sum of correlation over all weights
Wprev1[lyr][j][k]=W[lyr][j][k];
if (Epoch > 1) { //Wait till system is primed
P = 1/(1 + exp(ETA*DELTAc/T)); //Probability fn once it is
}
else
P= 0.5; //50% till then
R=(double)rand();
R=R/32767.0;
if (R>P) {
W[lyr][j][k]=W[lyr][j][k] - delta;
}
else {
W[lyr][j][k]=W[lyr][j][k] + delta;
} /* endif */
} /* endfor */
} /* endfor */
} /* endfor */
//Establish a new temp for the alopex probability fn base on avg correlation
if ((AnnealCount>= 1)&&(SumDELTAc>0.0)) {
if (SumDELTAc==10) {
printf("\nWarning SumDeltaC =0\n");
}
T=SumDELTAc/ AnnealCount;
AnnealCount=0;
SumDELTAc=0.0;
} else {
AnnealCount++;
} /* endif */
}
/******************************************************************************
* FUNCTION SaveWeights *
* *
* Output weights to default file DFLT.WGT *
******************************************************************************/
void ALOPEX::SaveWeights(char *WgtName) {
int lyr,s,d;
double zWT;
FILE *WEIGHTFILE;
WEIGHTFILE = fopen(WgtName,"w");
if (WEIGHTFILE==NULL){
printf("Unable to open weight file for output:%s\n",WgtName);
exit(0);
}
printf("SAVING CALCULATED WEIGHTS:\n\n");
fprintf(WEIGHTFILE,"0.00\n"); // Threshold always 0
fprintf(WEIGHTFILE,"%lf\n",Qzero); // Temperature
fprintf(WEIGHTFILE,"%d\n",NumLayers); // Number of layers
for (lyr=0; lyr<NumLayers; lyr++) { // Save topology
fprintf(WEIGHTFILE,"%d\n",LayerSize[lyr]); // Number of neurons/layer
}
for (lyr=1; lyr<NumLayers; lyr++) { // Start at 1st hidden
for (d=0; d<LayerSize[lyr]; d++) {
for (s=0; s<=LayerSize[lyr-1]; s++) { // One extra for bias
zWT=W[lyr][s][d];
fprintf(WEIGHTFILE,"%lf\n",zWT);
}
}
}
fclose(WEIGHTFILE);
}
/******************************************************************************
* FUNCTION Sigmoid *
* *
* Output non-linear squashing function *
******************************************************************************/
double ALOPEX::Sigmoid(double Net, double Tempr){
double x;
x = Net/Tempr;
if ( x >100.0)
return 1.0;
if ( x < -100.0)
return 0.0;
return 1.0/(1.0 + exp(-x));
}
/******************************************************************************
* FUNCTION train *
* Trains the network *
* Input: Training data file name and Parameter file name *
* Output:non-linear squashing function *
******************************************************************************/
int ALOPEX::train(char *TrnFname, char *ParmFname){
if (ArchOn) ARCHIVE=fopen("archive.lst","w");
GetParms(ParmFname);
GetTrnSet(TrnFname);
SetRandomWeights();
int Converged=0;
while ((!Converged) && (CurrIter < MAXITER) ) {
GetInputs();
RunNet();
RecordErrors();
if (CurrPat==(NumPatterns-1)) { // Test for full epoch
ArchivePut();
Converged=IsConverged();
AdaptWeights();
}
NextPat();
}
if (ArchOn) fclose(ARCHIVE);
return Converged;
}
//-----------------------------------------------------------------------------
/******************************************************************************
* FUNCTION ShowResults *
* *
* Output Summary specific to the parity-3 problem *
******************************************************************************/
void ShowResults(int ConvergeFlg, long int CurrEpoch, double temp){
printf("\n----------------------------------------------------------\n");
if (ConvergeFlg) {
printf("SUCCESS: Convergance has occured at iteration %ld\n",CurrEpoch);
}
else {
printf("FAILURE: Convergance has NOT occured!!\n");
} // endif
printf("Temperature = %lf\n",temp);
printf("\n----------------------------------------------------------\n");
}
ALOPEX Alopex;
/******************************************************************************
* MAIN *
******************************************************************************/
int main(int argc, char *argv[])
{
int Converged ;
if (argc>3) {
Converged = Alopex.train(argv[1],argv[2]);
}
else {
printf("USAGE: ALOPEX TRAINING_FILE PARMS_FILE WEIGHT_FILE\n");
exit(0);
}
// Show how we did
ShowResults(Converged, Alopex.QueryEpoch(), Alopex.QueryQzero());
if (Converged)
Alopex.SaveWeights(argv[3]); // Save the weights for later use
return 0;
}