// ======================================================================== // Copyright 2006 - Volian Enterprises, Inc. All rights reserved. // Volian Enterprises - Proprietary Information - DO NOT COPY OR DISTRIBUTE // ------------------------------------------------------------------------ // $Workfile: $ $Revision: $ // $Author: $ $Date: $ // // $History: $ // ======================================================================== using System; using System.Drawing; using System.Collections; using System.ComponentModel; using System.Windows.Forms; using System.Data; using System.Data.OleDb; using System.Collections.Specialized; using System.Collections.Generic; using System.Xml; using System.IO; using System.Text; using VEPROMS.CSLA.Library; namespace DataLoader { public partial class Loader { #region Log4Net public static readonly log4net.ILog _MyLog = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); //public static readonly log4net.ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); #endregion public static int HandleCount { get { return System.Diagnostics.Process.GetCurrentProcess().HandleCount; } } private bool RunWithDB2K = false; private Item MigrateProcedure(OleDbConnection cn, DataRow dr, Item FromItem, string pth, DocVersion docver, bool convertProcedures, FormatInfo activeFormat) { dicOldStepSequence = new Dictionary(); Stack SubSectLevels = new Stack(); // levels of subsections ProcFileName = dr["Entry"].ToString(); ProcNumber = dr["Number"].ToString(); frmMain.Status = "Processing Procedure " + ProcNumber + " - " + ProcFileName; GC.Collect(); frmMain.AddInfo("Processing Procedure {0} {1}", ProcNumber, GC.GetTotalMemory(true)); //frmMain.MyInfo = CSLACache.UsageNotFinalized; DateTime dts = GetDTS(dr["Date"].ToString(), dr["Time"].ToString()); string userid = dr["initials"].ToString().Trim(); if (userid == null || userid == "") userid = "Migration"; ConfigInfo ci = null; string tstr = dr["Proccode"].ToString(); if (tstr != null && tstr != "") { ci = new ConfigInfo(null); ci.AddItem("Procedure", "ProcCode", tstr); } //load revdate string tstr2 = dts.ToString("MM/dd/yyyy"); if (tstr2 != null && tstr2 != "") { if (ci == null) ci = new ConfigInfo(null); ci.AddItem("Procedure", "RevDate", tstr2); } if (ci == null) ci = new ConfigInfo(null); ci.AddItem("History", "ProcName", ProcFileName.ToUpper()); // Load in format data, i.e. default number of columns: string tstr1 = dr["Format"].ToString(); if (tstr1 != null && tstr1 != "") { switch (tstr1) { case "1": case "2": case "3": case "4": if (ci == null) ci = new ConfigInfo(null); ci.AddItem("Format", "Columns", tstr1); break; default: break; } } // try to load in fix file data: FileInfo fi = new FileInfo(pth + @"\" + ProcFileName + @".fix"); if (fi.Exists) { FixItems fixItems = new FixItems(fi); if (fixItems.Count > 0) { if (ci == null) ci = new ConfigInfo(null); if (fixItems[0].Rev != null && fixItems[0].Rev != "") ci.AddItem("Procedure", "Rev", fixItems[0].Rev); if (fixItems[0].RevDate != null && fixItems[0].RevDate != "") ci.AddItem("Procedure", "RevDate", fixItems[0].RevDate); if (fixItems[0].ReviewDate != null && fixItems[0].ReviewDate != "") ci.AddItem("Procedure", "ReviewDate", fixItems[0].ReviewDate); } } fi = new FileInfo(pth + @"\" + ProcFileName + @".apl"); if (fi.Exists) { MyProcAPL = GetApple(fi.FullName); } // check that there is a "Series" column, i.e. the 10th column, some datasets // may not have it, for example vetuec\master.sl1 if (dr.ItemArray.Length > 10) { tstr = dr["Series"].ToString(); if (tstr != null && tstr != "") { if (ci == null) ci = new ConfigInfo(null); ci.AddItem("Procedure", "Series", tstr); } } if (docver.DocVersionConfig.Unit_Count > 1) { for (int i = 1; i <= docver.DocVersionConfig.Unit_Count; i++) { ci.AddSlaveNode(i); docver.DocVersionConfig.SelectedSlave = i; string un = ".SL" + docver.DocVersionConfig.Old_Index; string mupth = pth.Replace(".prc", un); fi = new FileInfo(mupth + @"\" + ProcFileName + @".fix"); if (fi.Exists) { FixItems fixItems = new FixItems(fi); if (fixItems.Count > 0) { if (fixItems[0].Rev != null && fixItems[0].Rev != "") ci.AddSlaveItem(i, "Rev", fixItems[0].Rev); else ci.AddSlaveItem(i, "Rev", ""); if (fixItems[0].RevDate != null && fixItems[0].RevDate != "") ci.AddSlaveItem(i, "RevDate", fixItems[0].RevDate); else ci.AddSlaveItem(i, "RevDate", ""); if (fixItems[0].ReviewDate != null && fixItems[0].ReviewDate != "") ci.AddSlaveItem(i, "ReviewDate", fixItems[0].ReviewDate); else ci.AddSlaveItem(i, "ReviewDate", ""); } } else { ci.AddSlaveItem(i, "Rev", ""); ci.AddSlaveItem(i, "RevDate", ""); ci.AddSlaveItem(i, "ReviewDate", ""); } } } DataSet ds = new DataSet(); DataTable dt = null; // check that file exists, i.e. if proc file doesn't exist but has entry in set // file, flag error, but continue. string fname = pth + "\\" + dr["entry"] + ".dbf"; if (File.Exists(fname)) { // if the dbt is bad, fix it. DbtOk(dr["entry"].ToString(), pth); // See if there is PSI and if so, add it to the xml. OleDbDataAdapter dapsi = new OleDbDataAdapter("select * from [" + dr["entry"] + "] where [STEP] is null", cn); int handleBefore = HandleCount; TryToLoadDataSet(ds, fname, dapsi); int handleAfter = HandleCount; if (!RunWithDB2K && handleAfter - handleBefore > 10) { string msg = "It appears that handles are being created and not cleared."; if (dB2KInstalled()) { msg += "\r\n\r\nThis is most likely because dB2K is installed." + "\r\n\r\nUninstall dB2K and run again."; if (MessageBox.Show(msg + "\r\n\r\nDo you want to continue?", "Run with dB2K", MessageBoxButtons.YesNo, MessageBoxIcon.Question) == DialogResult.Yes) RunWithDB2K = true; } else { msg += "\r\n\r\nCall Volian for support."; } if (!RunWithDB2K) throw new Exception(msg); } dt = ds.Tables[0]; if (dt.Rows.Count > 0) { DataRow drpsi = dt.Rows[0]; string psistr = drpsi["TEXTM"].ToString(); try { if (psistr != null && psistr != "") { StringReader strrd = new StringReader(psistr); string sLine; if (ci == null) ci = new ConfigInfo(null); while ((sLine = strrd.ReadLine()) != null) { int indx = sLine.IndexOf(' '); string nm = null; string vl = null; if (indx < 0) nm = sLine; else { nm = sLine.Substring(0, indx); string tmpPsi = sLine.Substring(indx + 1, sLine.Length - indx - 1); if (tmpPsi.Length > 0) tmpPsi = TextConvert.ConvertText(tmpPsi); vl = tmpPsi; } ci.AddItem("PSI", nm, vl == null ? null : vl); } } } catch(Exception ex) { log.Warn("Failure during PSI", ex); } } dapsi.Dispose(); } else // log an error { log.ErrorFormat("Missing DBF: {0}", fname); } // Note, for now the data from the format field will be saved. Later, xpath, ?? EditSectId = 0; // See if no caret convert... //FormatInfo docverFormat = docver.MyDocVersionInfo.ActiveFormat; bool do_cvt = activeFormat.PlantFormat.FormatData.SectData.ConvertCaretToDelta;//docverFormat.PlantFormat.FormatData.SectData.ConvertCaretToDelta; // pass in a 0 on MakeContent for type of procedure, i.e. procedure = 0 (on type field) // can we save itm.ItemID in a dictionary of outside trans? ex: otrans("vehlp\procs\0POP05-EO00",itm.ItemID)? Content cont = null; Item itm = null; string dicNumber = dr["Number"].ToString(); string number = TextConvert.ConvertText(dr["Number"].ToString()); frmMain.UpdateLabels(1, 0, 0); //Console.WriteLine("number {0}", number); //Console.WriteLine("Item Key {0} List {1}", Item.CacheCountPrimaryKey, Item.CacheCountList); //Console.WriteLine("ItemInfo Key {0} List {1}", ItemInfo.CacheCountPrimaryKey, ItemInfo.CacheCountList); DocVersionInfo dvi = DocVersionInfo.Get(docver.VersionID); dvi.ResetProcedures(); // need this to clear out Procedures so that they are retrieved (resolves a caching problem) foreach (ItemInfo ii in dvi.Procedures) { if (ii.MyContent.Number == number) { //Console.WriteLine(number); itm = ii.Get(); cont = ii.MyContent.Get(); break; } } if (!convertProcedures || (docver.VersionType != (int)VEPROMS.CSLA.Library.VersionTypeEnum.WorkingDraft && docver.VersionType != (int)VEPROMS.CSLA.Library.VersionTypeEnum.Approved)) { // if this procedure already exists for this docversion, just use it. if (itm == null && cont == null) { cont = Content.New(number, TextConvert.ConvertText(dr["Title"].ToString(), do_cvt), 0, null, ci == null ? null : ci.ToString(), dts, userid); itm = Item.MakeItem(FromItem, cont, dts, userid); } } if (convertProcedures /* && number == "001\\u8209?007"/* && number == "0POP05-EO-EC00"*/) { OleDbDataAdapter da = new OleDbDataAdapter("select * from [" + dr["entry"] + "] where sequence like ' %' and [STEP] is not null", cn); try { dicTrans_ItemDone[dicNumber] = itm; // will be null if only doing procs & no sections, etc. LoadSection(ds, da, dr["entry"].ToString()); da.SelectCommand.CommandText = "select * from [" + dr["entry"] + "] where step not like ' ' and sequence not like ' %'"; da.Fill(ds, "Steps"); dt = ds.Tables["Steps"]; dt.CaseSensitive = true; ds.Tables["Steps"].CaseSensitive = true; dt.Columns.Add("CStep", System.Type.GetType("System.String")); dt.Columns.Add("CSequence", System.Type.GetType("System.String")); // set the cstep & csequence - couldn't do it in the add because it needed a sql function foreach (DataRow drw in ds.Tables["Steps"].Rows) { drw["CStep"] = TextConvert.ConvertSeq(drw["Step"].ToString()); drw["CSequence"] = TextConvert.ConvertSeq(drw["Sequence"].ToString()); } dt.Columns.Add("StepNo", System.Type.GetType("System.Int32"), "Convert(Convert(Substring(CStep,2,1),'System.Char'),'System.Int32')-48"); dt.Columns.Add("Level", System.Type.GetType("System.Int32"), "Len(CSequence)"); dt.Columns.Add("SubStepNo", System.Type.GetType("System.Int32"), "Convert(Convert(Substring(CSequence,Len(CSequence),1),'System.Char'),'System.Int32')-48"); } catch (Exception ex) { log.ErrorFormat("{0}\r\n\r\n{1}", ex.Message, ex.InnerException); log.Error(ex.StackTrace); frmMain.AddError(ex, "MigrateProcedure - {0}", ProcFileName); } da.Dispose(); // dicSecCount is dictionary to track number of subsections for the parent at a level // dicSecID is the parent at that level Dictionary dicSecCount = new Dictionary(); Dictionary dicSecParentItem = new Dictionary(); frmMain.pbSectMaximum = ds.Tables["Sections"].Rows.Count; frmMain.pbSectValue = 0; DataTable dtsect = ds.Tables["Sections"]; dtsect.CaseSensitive = true; DataView dv = new DataView(dtsect, "", "locb", DataViewRowState.CurrentRows); Item SectItm = null; int level = 0; bool addpart = true; Item parentitem = itm; //if (ProcNumber == "001-002") // Console.WriteLine("proc 001-002"); foreach (DataRowView drw in dv) { //if (ProcNumber == "016-001") //"017-001") //"082-002AB") // Console.WriteLine("016-001"); //("017-001"); //"082-002AB"); // If there is a '~' in step[1] and the sequence number is blank, the this record // is a multiline section title. Skip it from processing the section. This record // will get read in the section code to get the section title. string sequence = drw["CSequence"].ToString().PadRight(10); string step = drw["CStep"].ToString(); if (step[1]=='~' && sequence[1]==' ') break; bool hasSubSects = (sequence[5] != ' ' && sequence[5] != '0'); SectItm = MigrateSection(parentitem, ProcNumber, cn, drw, ds.Tables["Steps"], SectItm, hasSubSects, pth, docver, activeFormat, addpart ? cont : null); addpart = false; // if no children, add first child (cont) //if (addpart) //{ // // ContentsParts.Add can use 'fromtype', item - fromtype here = 2, section // cont.ContentParts.Add(2, SectItm); // if (cont.MyZContent.OldStepSequence == null || cont.MyZContent.OldStepSequence == "") cont.MyZContent.OldStepSequence = ProcNumber; // if (!cont.IsSavable) ErrorRpt.ErrorReport(cont); // cont.Save(); // addpart = false; //} //FrType = 0; //Console.WriteLine("'before',{0},{1},{2},{3}", SectItm == null ? "" : SectItm.DisplayNumber, parentitem.DisplayNumber, level, dicSecCount.Count > 0 ? dicSecCount[level] : 0); dicSecParentItem[level] = SectItm; int subSecs = drw["Sequence"].ToString().PadRight(12, ' ')[5] - 48; if (level > 0) { dicSecCount[level]--; // decrements number of sections to process } if (subSecs > 0) { dicSecCount[++level] = subSecs; cont = SectItm.MyContent; parentitem = SectItm; addpart = true; SectItm = null; // no previous sibling for the first child node. } if (level > 0) { while (level > 0 && dicSecCount[level] == 0) { // shift up a level, i.e. give me the parent for the previous level dicSecCount.Remove(level); SectItm = dicSecParentItem[--level]; if (level > 0) parentitem = dicSecParentItem[level - 1]; else parentitem = itm; } } //Console.WriteLine("'after',{0},{1},{2},{3}", SectItm == null ? "" : SectItm.DisplayNumber, parentitem.DisplayNumber, level, dicSecCount.Count > 0 ? dicSecCount[level] : 0); } dv.Dispose(); //foreach (Item secItm in dicSecParentItem.Values) // secItm.Dispose(); // need section start if (EditSectId != 0) { if (ci == null) ci = new ConfigInfo(null); ci.AddItem("Procedure", "SectionStart", string.Format("{0}", EditSectId)); itm.MyContent.Config = (ci == null) ? null : ci.ToString(); if (!itm.IsSavable) ErrorRpt.ErrorReport(itm); itm.Save(); } } ds.Dispose(); // if there are any floating foldouts & associated steps, set config on steps for foldout association. // the free the lists for the next procedure: if (FoldoutStpSeq != null && FoldoutStpSeq.Count>0) { string cmd = "SELECT * FROM [tran] WHERE [FROMNUMBER]='" + ProcNumber.Replace("'", "''") + "' AND [TYPE] ='R'"; OleDbDataAdapter da = new OleDbDataAdapter(cmd, cn); // get transition records for this step. ds = new DataSet(); try { da.Fill(ds); DataTable dtt = ds.Tables[0]; dtt.CaseSensitive = true; Dictionary stpFoldSeq = new Dictionary(); foreach (DataRow drt in dtt.Rows) { // get from/to seq, subtract off '0' to get stepnumber. // The 'oldto' field stores which foldout section these steps go // to. However, if reset, the 'oldto' may not contain a reference // to a foldout, i.e. may contain 'junk' data. Only add the // reference to the dictionary if it's a foldout. int fromStep = TextConvert.ConvertSeq(drt["FROMSEQUEN"].ToString())[1] - '0'; int toStep = TextConvert.ConvertSeq(drt["TOSEQUENCE"].ToString())[1] - '0'; string foldSeq = drt["OLDTO"].ToString().Substring(0, 2); if (foldSeq[1] == 'F') { for (int istp = fromStep; istp <= toStep; istp++) stpFoldSeq.Add(istp, foldSeq); } } ds.Dispose(); da.Dispose(); // now read in the HLS's and set their config to the appropriate foldout. // stpFoldSeq contains 16 bit step to foldout associations, i.e. an integer step // number (ordinal) and string sequence of foldout. // FloatingFoldoutHLS contains each 16bit ordinal step number (all steps, whether // they have a foldout or not) and the associated itemid in 32bit // FoldoutStpSeq contains all foldouts, i.e. the string representing the sequence // number if 16bit and the resulting 32bit itemid. foreach (KeyValuePair kvp in stpFoldSeq) { int hlsItemId = FloatingFoldoutHLS[kvp.Key-1]; // -1 for indexes starting at 0 (not 1) using (Item itmF = Item.Get(hlsItemId)) { // be sure that the dictionaries contain the key. // Some data may be 'corrupt' and not have the appropriate linkages // to floating foldouts. if (FoldoutStpSeq.ContainsKey(kvp.Value)) { int fldItemId = FoldoutStpSeq[kvp.Value]; ConfigInfo ciFold = new ConfigInfo(itmF.MyContent.Config); ciFold.AddItem("Step", "FloatingFoldout", fldItemId.ToString()); itmF.MyContent.Config = ciFold.ToString(); itmF.Save(); } } } } catch (Exception ex) { frmMain.AddError(ex, "Error resolving Floating Foldout Data '{0}'", ProcNumber); log.Error("Error resolving Floating Foldout Data"); log.ErrorFormat("from number = {0}", ProcNumber); log.ErrorFormat("{0}\r\n\r\n{1}", ex.Message, ex.InnerException); ds.Dispose(); da.Dispose(); } } if (FoldoutStpSeq!=null) FoldoutStpSeq.Clear(); if (FloatingFoldoutHLS!=null) FloatingFoldoutHLS.Clear(); //Console.WriteLine("End MigrateProcedure number {0}", number); //Console.WriteLine("End MigrateProcedure Item Key {0} List {1}", Item.CacheCountPrimaryKey, Item.CacheCountList); //Console.WriteLine("End MigrateProcedure ItemInfo Key {0} List {1}", ItemInfo.CacheCountPrimaryKey, ItemInfo.CacheCountList); return itm; } private static void TryToLoadDataSet(DataSet ds, string fname, OleDbDataAdapter dapsi) { try { dapsi.Fill(ds); } catch (Exception ex) { FileInfo fi; switch (ex.Message) { case "Index file not found.":// then delete inf file fi = new FileInfo(fname.ToLower().Replace(".dbf",".inf")); fi.Delete(); TryToLoadDataSet(ds, fname, dapsi); // Try again break; case "External table is not in the expected format.": // then pad dbt file with 128 zeros. fi = new FileInfo(fname.ToLower().Replace(".dbf", ".dbt")); FileStream fs = fi.OpenWrite(); fs.Position = fs.Length; byte[] buf = new byte[128]; for (int i = 0; i < 128; i++) buf[i] = 0; fs.Write(buf, 0, 128); fs.Close(); TryToLoadDataSet(ds, fname, dapsi); // Try again break; default: // Unrecognized error Console.WriteLine(ex.Message); //Application.Exit(); throw new Exception("Error in MigrateProcedure: " + fname, ex); } } } private bool dB2KInstalled() { Microsoft.Win32.RegistryKey key = Microsoft.Win32.Registry.LocalMachine.OpenSubKey(@"software\dBase\dB2K"); bool retval = key != null; if(key != null)key.Close(); return retval; } // check for duplicate SET file data - jsj 2/11/10 private bool OKtoProcessDBF(DataRow dr, string pth) { string pfn = dr["Entry"].ToString(); string pn = dr["Number"].ToString(); FileInfo fi = new FileInfo(pth + "\\" + pfn + ".dbf"); if (!fi.Exists) { _MyLog.WarnFormat("Data file does not exist {0}", fi.FullName); frmMain.AddError("Data file does not exist {0}", fi.FullName); return false; } if (!dicSetfileEntries.ContainsValue(pfn) && !dicSetfileEntries.ContainsKey(pn)) { dicSetfileEntries.Add(pn, pfn); return true; } return false; } private Item MigrateProcedures(OleDbConnection cn, string pth, DocVersion docver, bool convertProcedures, FormatInfo activeFormat) { Item FirstItm = docver.MyItem;// null; // Loop through Set File for each Procedure try { OleDbDataAdapter da = new OleDbDataAdapter("Select * from [set] where entry is not null", cn); DataSet ds = new DataSet(); da.Fill(ds); Item FrItm = null; if (FirstItm != null) FrItm = ItemInfo.Get(FirstItm.ItemID).LastSibling.Get(); frmMain.pbProcMaximum = ds.Tables[0].Rows.Count; frmMain.UpdateLabels(0, 0, 0); dicSetfileEntries = new Dictionary(); // used to check for duplicate SET file info - jsj 2/11/10 int i = 0; frmMain.ShowSections = convertProcedures; foreach (DataRow dr in ds.Tables[0].Rows) { i++; if (i > frmMain.SkipProcedures && (frmMain.HowManyProcedures == 0|| i <= (frmMain.SkipProcedures + frmMain.HowManyProcedures) )) { if (OKtoProcessDBF(dr, pth)) // look for duplicate SET file info - jsj 2/11/10 { List cacheContentInfo = ContentInfo.CacheList; List cacheItemInfo = ItemInfo.CacheList; List cacheEntryInfo = EntryInfo.CacheList; List cachePdfInfo = PdfInfo.CacheList; List cacheDocVersionInfo = DocVersionInfo.CacheList; List cachePartInfo = PartInfo.CacheList; frmMain.pbStepValue = 0;//Reset Step Count FrItm = MigrateProcedure(cn, dr, FrItm, pth, docver, convertProcedures, activeFormat); if (FirstItm == null) FirstItm = FrItm; //frmMain.MyInfo = string.Format("Before Restore {0}", GC.GetTotalMemory(true)); ContentInfo.RestoreCacheList(cacheContentInfo); PartInfo.RestoreCacheList(cachePartInfo); ItemInfo.RestoreCacheList(cacheItemInfo); EntryInfo.RestoreCacheList(cacheEntryInfo); PdfInfo.RestoreCacheList(cachePdfInfo); DocVersionInfo.RestoreCacheList(cacheDocVersionInfo); //frmMain.MyInfo = string.Format("After Restore {0}", GC.GetTotalMemory(true)); } } //GC.Collect(); } ds.Dispose(); da.Dispose(); } catch (Exception ex) { _MyLog.ErrorFormat("MigrateProcedures - {0} - {1}\r\n\r\n{2}", pth,ex.Message, ex.InnerException); //throw new Exception("Error in MigrateProcedures: " + pth, ex); } frmMain.MyInfo = CSLACache.UsageAll; frmMain.AddInfo(Content.MyStack); frmMain.AddInfo(ContentInfo.MyStack); frmMain.AddInfo(DocVersionInfo.MyStack); return FirstItm; } private void DbtOk(string fname, string pth) { string dbtname = pth + "\\" + fname + ".dbt"; if (File.Exists(dbtname)) { // check if dbt is at least 512, if not append 508 bytes. This is to fix // a problem where dbts were created with only 4 bytes in the 16-bit code. // if only 4 bytes, ado.net gives an "External table not in expected format" // error FileInfo fi = new FileInfo(dbtname); if (fi.Length < 512) { FileStream fs = new FileStream(dbtname, FileMode.Open, FileAccess.Write, FileShare.ReadWrite); BinaryWriter bw = new BinaryWriter(fs); bw.Seek(0, SeekOrigin.End); byte[] wrBytes = new byte[512 - fi.Length]; for (int i = 0; i < 512 - fi.Length; i++) wrBytes[i] = 0; wrBytes[4] = 0x02; bw.Write(wrBytes); bw.Close(); } fi = null; } } } }