michael boeni

Dear all

but I get a severe memory problem. The tool should import around 250
text files, with the largest being 70 Megs.

My problem is that the application does not seem to manage memory very
well or I have introduced a memory leak somewhere. This snipped of code causes the problem:
<---Code Snippet------------------------------------------------------>
outputArea.append("Importing...:");
database1.openConnection();
for (int i = 0; i < fileNames.length; i++)
{

String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
textDataFile.setFileName(absolutePath);
textDataFile.load(tableDataSet1);
System.out.println("Current file: " + absolutePath);
outputArea.append("\nRows loaded: " + tableDataSet1.getRowCount());
database1.saveChanges(tableDataSet1);
tableDataSet1.clearValues();

}
database1.closeConnection(); outputArea.append("\nImport completed!");
<---Code Snippet------------------------------------------------------>

Does anyone know what I am doing wrong here? How can I improve memory
consumption?

Here is the full class:
<---Code Snippet------------------------------------------------------>
package com.shiftthink.geotools.data.geonames;

import java.io.*;
import com.borland.dx.dataset.*;
import com.borland.dx.sql.dataset.*;
import javax.swing.JTextArea;
import javax.swing.JProgressBar;
import java.util.Arrays;

public class DBHelper
{
static String[] fileNames =
{
"ac", "al", "ag", "an", "ao", "av", "af", "ar", "am", "aa", "at",
"as",
"au", "aj", "bf", "ba", "bg", "bb", "bs", "bo", "be", "bh", "bn",
"bd",
"bt", "bl", "bk", "bc", "bv", "br", "io", "vi", "bx", "bu", "uv",
"bm",
"by", "cb", "cm", "ca", "cv", "cj", "ct", "cd", "ci", "ch", "kt",
"ip",
"ck", "co", "cn", "cf", "cg", "cw", "cr", "cs", "iv", "hr", "cu",
"cy",
"ez", "da", "dj", "do", "dr", "tt", "ec", "eg", "es", "ek", "er",
"en",
"et", "eu", "fk", "fo", "fj", "fi", "fr", "fg", "fp", "fs", "gb",
"ga",
"gz", "gg", "gm", "gh", "gi", "go", "gr", "gl", "gj", "gp", "gt",
"gt",
"gk", "pu", "gy", "ha", "hm", "ho", "hk", "hu", "ic", "in", "id",
"ir",
"iz", "ei", "im", "is", "it", "jm", "jn", "ja", "je", "jo", "ju",
"kz",
"ke", "kr", "ku", "kg", "la", "lg", "le", "lt", "li", "ly", "ls",
"lh",
"lu", "mc", "mk", "ma", "mi", "my", "mv", "ml", "mt", "rm", "mb",
"mr",
"mp", "mf", "mx", "fm", "md", "mn", "mg", "mh", "mo", "mz", "wa",
"nr",
"np", "nl", "nt", "nc", "nz", "nu", "ng", "ni", "ne", "nm", "nf",
"kn",
"no", "os", "mu", "pk", "ps", "pm", "pp", "pf", "pa", "pe", "rp",
"pc",
"pl", "po", "qa", "re", "ro", "rs", "rw", "sh", "sc", "st", "sb",
"vc",
"ws", "sm", "tp", "sa", "sg", "yi", "se", "sl", "sn", "lo", "si",
"bp",
"so", "sf", "sx", "ks", "sp", "pg", "ce", "su", "ns", "sv", "wz",
"sw",
"sz", "sy", "tw", "ti", "tz", "th", "to", "tl", "tn", "td", "te",
"ts",
"tu", "tx", "tk", "tv", "ug", "up", "uf", "ae", "uk", "uy", "uz",
"nh",
"vt", "ve", "vm", "wf", "we", "wi", "ym", "za", "zi"
};
TextDataFile textDataFile = new TextDataFile();
TableDataSet tableDataSet1 = new TableDataSet();
Column RC = new Column();
Column UFI = new Column();
Column UNI = new Column();
Column LAT = new Column();
Column LONG = new Column();
Column DMS_LAT = new Column();
Column DMS_LONG = new Column();
Column UTM = new Column();
Column JOG = new Column();
Column FC = new Column();
Column DSG = new Column();
Column PC = new Column();
Column CC1 = new Column();
Column ADM1 = new Column();
Column ADM2 = new Column();
Column DIM = new Column();
Column CC2 = new Column();
Column NT = new Column();
Column LC = new Column();
Column SHORT_FORM = new Column();
Column GENERIC = new Column();
Column SHORT_NAME = new Column();
Column FULL_NAME = new Column();
Column FULL_NAME_ND = new Column();
Column MOD_DATE = new Column();
Database database1 = new Database();


public DBHelper()
{

}


public void gnImport(String filePath, JTextArea outputArea) throws
Exception
{
database1.setConnection(new
com.borland.dx.sql.dataset.ConnectionDescriptor("j dbcostgresql:ST_EIS",
"st_eis_user", "fairline", false, "org.postgresql.Driver"));
MOD_DATE.setCaption("MOD_DATE");
MOD_DATE.setColumnName("MOD_DATE");
MOD_DATE.setDataType(com.borland.dx.dataset.Varian t.STRING);
MOD_DATE.setPreferredOrdinal(24);
MOD_DATE.setWidth(15);
MOD_DATE.setServerColumnName("MOD_DATE");
MOD_DATE.setSqlType(0);
FULL_NAME_ND.setCaption("FULL_NAME_ND");
FULL_NAME_ND.setColumnName("FULL_NAME_ND");
FULL_NAME_ND.setDataType(com.borland.dx.dataset.Va riant.STRING);
FULL_NAME_ND.setPreferredOrdinal(23);
FULL_NAME_ND.setWidth(200);
FULL_NAME_ND.setServerColumnName("FULL_NAME_ND");
FULL_NAME_ND.setSqlType(0);
FULL_NAME.setCaption("FULL_NAME");
FULL_NAME.setColumnName("FULL_NAME");
FULL_NAME.setDataType(com.borland.dx.dataset.Varia nt.STRING);
FULL_NAME.setPreferredOrdinal(22);
FULL_NAME.setWidth(200);
FULL_NAME.setServerColumnName("FULL_NAME");
FULL_NAME.setSqlType(0);
SHORT_NAME.setCaption("SHORT_NAME");
SHORT_NAME.setColumnName("SHORT_NAME");
SHORT_NAME.setDataType(com.borland.dx.dataset.Vari ant.STRING);
SHORT_NAME.setPreferredOrdinal(21);
SHORT_NAME.setWidth(200);
SHORT_NAME.setServerColumnName("SHORT_NAME");
SHORT_NAME.setSqlType(0);
GENERIC.setCaption("GENERIC");
GENERIC.setColumnName("GENERIC");
GENERIC.setDataType(com.borland.dx.dataset.Variant .STRING);
GENERIC.setPreferredOrdinal(20);
GENERIC.setWidth(128);
GENERIC.setServerColumnName("GENERIC");
GENERIC.setSqlType(0);
SHORT_FORM.setCaption("SHORT_FORM");
SHORT_FORM.setColumnName("SHORT_FORM");
SHORT_FORM.setDataType(com.borland.dx.dataset.Vari ant.STRING);
SHORT_FORM.setPreferredOrdinal(19);
SHORT_FORM.setWidth(128);
SHORT_FORM.setServerColumnName("SHORT_FORM");
SHORT_FORM.setSqlType(0);
LC.setCaption("LC");
LC.setColumnName("LC");
LC.setDataType(com.borland.dx.dataset.Variant.STRI NG);
LC.setPreferredOrdinal(18);
LC.setWidth(2);
LC.setServerColumnName("LC");
LC.setSqlType(0);
NT.setCaption("NT");
NT.setColumnName("NT");
NT.setDataType(com.borland.dx.dataset.Variant.STRI NG);
NT.setPreferredOrdinal(17);
NT.setWidth(1);
NT.setServerColumnName("NT");
NT.setSqlType(0);
CC2.setCaption("CC2");
CC2.setColumnName("CC2");
CC2.setDataType(com.borland.dx.dataset.Variant.STR ING);
CC2.setPreferredOrdinal(16);
CC2.setWidth(3);
CC2.setServerColumnName("CC2");
CC2.setSqlType(0);
DIM.setCaption("DIM");
DIM.setColumnName("DIM");
DIM.setDataType(com.borland.dx.dataset.Variant.STR ING);
DIM.setPreferredOrdinal(15);
DIM.setWidth(15);
DIM.setServerColumnName("DIM");
DIM.setSqlType(0);
ADM2.setCaption("ADM2");
ADM2.setColumnName("ADM2");
ADM2.setDataType(com.borland.dx.dataset.Variant.ST RING);
ADM2.setPreferredOrdinal(14);
ADM2.setWidth(200);
ADM2.setServerColumnName("ADM2");
ADM2.setSqlType(0);
ADM1.setCaption("ADM1");
ADM1.setColumnName("ADM1");
ADM1.setDataType(com.borland.dx.dataset.Variant.ST RING);
ADM1.setPreferredOrdinal(13);
ADM1.setWidth(2);
ADM1.setServerColumnName("ADM1");
ADM1.setSqlType(0);
CC1.setCaption("CC1");
CC1.setColumnName("CC1");
CC1.setDataType(com.borland.dx.dataset.Variant.STR ING);
CC1.setPreferredOrdinal(12);
CC1.setWidth(3);
CC1.setServerColumnName("CC1");
CC1.setSqlType(0);
PC.setCaption("PC");
PC.setColumnName("PC");
PC.setDataType(com.borland.dx.dataset.Variant.STRI NG);
PC.setPreferredOrdinal(11);
PC.setWidth(2);
PC.setServerColumnName("PC");
PC.setSqlType(0);
DSG.setCaption("DSG");
DSG.setColumnName("DSG");
DSG.setDataType(com.borland.dx.dataset.Variant.STR ING);
DSG.setPreferredOrdinal(10);
DSG.setWidth(12);
DSG.setServerColumnName("DSG");
DSG.setSqlType(0);
FC.setCaption("FC");
FC.setColumnName("FC");
FC.setDataType(com.borland.dx.dataset.Variant.STRI NG);
FC.setPreferredOrdinal(9);
FC.setWidth(2);
FC.setServerColumnName("FC");
FC.setSqlType(0);
JOG.setCaption("JOG");
JOG.setColumnName("JOG");
JOG.setDataType(com.borland.dx.dataset.Variant.STR ING);
JOG.setPreferredOrdinal(8);
JOG.setWidth(10);
JOG.setServerColumnName("JOG");
JOG.setSqlType(0);
UTM.setCaption("UTM");
UTM.setColumnName("UTM");
UTM.setDataType(com.borland.dx.dataset.Variant.STR ING);
UTM.setPreferredOrdinal(7);
UTM.setWidth(5);
UTM.setServerColumnName("UTM");
UTM.setSqlType(0);
DMS_LONG.setCaption("DMS_LONG");
DMS_LONG.setColumnName("DMS_LONG");
DMS_LONG.setDataType(com.borland.dx.dataset.Varian t.STRING);
DMS_LONG.setPreferredOrdinal(6);
DMS_LONG.setServerColumnName("DMS_LONG");
DMS_LONG.setWidth(15);
DMS_LONG.setSqlType(0);
DMS_LAT.setCaption("DMS_LAT");
DMS_LAT.setColumnName("DMS_LAT");
DMS_LAT.setDataType(com.borland.dx.dataset.Variant .STRING);
DMS_LAT.setPreferredOrdinal(5);
DMS_LAT.setServerColumnName("DMS_LAT");
DMS_LAT.setWidth(15);
DMS_LAT.setSqlType(0);
LONG.setCaption("LONG");
LONG.setColumnName("LONG");
LONG.setDataType(com.borland.dx.dataset.Variant.ST RING);
LONG.setPreferredOrdinal(4);
LONG.setServerColumnName("LONG");
LONG.setWidth(15);
LONG.setSqlType(0);
LAT.setCaption("LAT");
LAT.setColumnName("LAT");
LAT.setDataType(com.borland.dx.dataset.Variant.STR ING);
LAT.setPreferredOrdinal(3);
LAT.setServerColumnName("LAT");
LAT.setWidth(15);
LAT.setSqlType(0);
UNI.setCaption("UNI");
UNI.setColumnName("UNI");
UNI.setDataType(com.borland.dx.dataset.Variant.STR ING);
UNI.setPreferredOrdinal(2);
UNI.setServerColumnName("UNI");
UNI.setWidth(15);
UNI.setSqlType(0);
UFI.setCaption("UFI");
UFI.setColumnName("UFI");
UFI.setDataType(com.borland.dx.dataset.Variant.STR ING);
UFI.setPreferredOrdinal(1);
UFI.setServerColumnName("UFI");
UFI.setWidth(15);
UFI.setSqlType(0);
RC.setCaption("RC");
RC.setColumnName("RC");
RC.setDataType(com.borland.dx.dataset.Variant.STRI NG);
RC.setPreferredOrdinal(0);
RC.setWidth(1);
RC.setServerColumnName("RC");
RC.setSqlType(0);


tableDataSet1.setColumns(new Column[]
{RC, UFI, UNI, LAT, LONG, DMS_LAT,
DMS_LONG, UTM,
JOG, FC, DSG, PC, CC1, ADM1, ADM2, DIM,
CC2, NT,
LC, SHORT_FORM, GENERIC, SHORT_NAME, FULL_NAME,
FULL_NAME_ND, MOD_DATE});
tableDataSet1.setTableName("coordinates");
tableDataSet1.setSchemaName("st_eis");

textDataFile.setDelimiter("");
textDataFile.setEncoding("Cp1252");
textDataFile.setFileFormat(com.borland.dx.dataset. DataFileFormat.ASCII);

textDataFile.setSeparator("\t");
textDataFile.setLoadAsInserted(true);

try
{

/**
================================================== ================*/
/** Removing first line (headers) */
/**
================================================== ================*/
for (int i = 0; i < fileNames.length; i++)
{
String removerPath = filePath + "\\" + fileNames[i] + ".txt";
RandomAccessFile file = new RandomAccessFile(removerPath, "rws");
file.seek(0);
String line1 = file.readLine();
byte content[] = new byte[line1.length()];
file.seek(0);
file.write(content);
file.close();
}

// ================================================== ================
// Importing
// ================================================== ================
outputArea.append("Importing...:");
database1.openConnection();
for (int i = 0; i < fileNames.length; i++)
{

String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
textDataFile.setFileName(absolutePath);
textDataFile.load(tableDataSet1);
System.out.println("Current file: " + absolutePath);
outputArea.append("\nRows loaded: " + tableDataSet1.getRowCount());
database1.saveChanges(tableDataSet1);
tableDataSet1.clearValues();

}
database1.closeConnection();
outputArea.append("\nImport completed!");
}

catch (Exception ex)
{
ex.printStackTrace();
throw new Exception();
}

} }
<---Code Snippet------------------------------------------------------>

Thanks,
Michael

john moore

Michael,

Other than being a "very wide" table and therefore a LOT of column
objects.. Nothing is obviously off..

Have you tried giving the VM more memory..??

John..

--
=============================================
TeamB are volunteer helpers. Please DO NOT REPLY VIA EMAIL!
Post all questions and replies to this newsgroup ONLY
For papers on DataExpress, Applets, JSP, and Web Development go to:
http://www.microps.com/mps/paperFAQ.html
================================================== ==

michael boeni

Hi John

Yes I did. I used "XmX700" which gives java a heap size of up to this
amount. But thats crazy, I still get the out of memory error. I am
really flattered on how this can be...Even forcing a GC does not do
anything good...

Cheers,
Michael

john moore

Well the next step would be to run this against OptimizeIt or some other
memory profiler so that you can narrow down were the "villian" is..

John..

--
=============================================
TeamB are volunteer helpers. Please DO NOT REPLY VIA EMAIL!
Post all questions and replies to this newsgroup ONLY
For papers on DataExpress, Applets, JSP, and Web Development go to:
http://www.microps.com/mps/paperFAQ.html
================================================== ==

michael boeni

Hi John

I did that ant get strange results. According to Optimizeit, the Heap is
regularily emptied and then memory consumption mounts again - which is
as I expected. The strange thing is that the memory consumed as shown by
Windows only goes upward, until Win starts swapping...

I cannot believe that the DataExpress components are so extremely
memory-inefficient. There must be a way to improve that...

john moore

If you can...

Produce a "small" demo that trips this problem.. Provide repeatable
instructions..

email it to me to verify..

If I can duplicate it, I will pass it on to the "right" persons..

John..


--
=============================================
TeamB are volunteer helpers. Please DO NOT REPLY VIA EMAIL!
Post all questions and replies to this newsgroup ONLY
For papers on DataExpress, Applets, JSP, and Web Development go to:
http://www.microps.com/mps/paperFAQ.html
================================================== ==