ConcatTable.h

Go to the documentation of this file.

// # ConcatTable.h: Class to view a concatenation of tables as a single table
// # Copyright (C) 2008
// # Associated Universities, Inc. Washington DC, USA.
// #
// # This library is free software; you can redistribute it and/or modify it
// # under the terms of the GNU Library General Public License as published by
// # the Free Software Foundation; either version 2 of the License, or (at your
// # option) any later version.
// #
// # This library is distributed in the hope that it will be useful, but WITHOUT
// # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
// # License for more details.
// #
// # You should have received a copy of the GNU Library General Public License
// # along with this library; if not, write to the Free Software Foundation,
// # Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
// #
// # Correspondence concerning AIPS++ should be addressed as follows:
// #        Internet email: casa-feedback@nrao.edu.
// #        Postal address: AIPS++ Project Office
// #                        National Radio Astronomy Observatory
// #                        520 Edgemont Road
// #                        Charlottesville, VA 22903-2475 USA
 
#ifndef TABLES_CONCATTABLE_H
#define TABLES_CONCATTABLE_H
 
// # Includes
#include <casacore/casa/aips.h>
#include <casacore/tables/Tables/BaseTable.h>
#include <casacore/tables/Tables/ConcatRows.h>
#include <casacore/tables/Tables/TableRecord.h>
#include <casacore/tables/Tables/Table.h>
#include <casacore/casa/BasicSL/String.h>
#include <casacore/casa/Arrays/Vector.h>
#include <map>
 
namespace casacore {  // # NAMESPACE CASACORE - BEGIN
 
// # Forward Declarations
class TSMOption;
class ConcatColumn;
class AipsIO;
 
// <summary>
// Class to view a concatenation of tables as a single table.
// </summary>
 
// <use visibility=local>
 
// <reviewed reviewer="UNKNOWN" date="" tests="">
// </reviewed>
 
// <prerequisite>
// # Classes you should understand before using this one.
//   <li> BaseTable
//   <li> ConcatColumn
// </prerequisite>
 
// <etymology>
// ConcatTable represents the concatenation of one or more tables.
// </etymology>
 
// <synopsis>
// ConcatTable is used to virtually concatenate one or more tables.
// Those tables must have the same description.
//
// It acts to the user as a normal table. All gets and puts are
// handled by ConcatColumn which directs them to the referenced columns
// while (if needed) converting the given row number to the row number
// in the referenced tables. For that purpose ConcatTable keeps the
// number of rows in the referenced tables.
// <note>Currently it cannot handle changes in the number of rows in the
// underlying tables. </note>
//
// It is possible to specify the keyword names of the subtables that have
// to be concatenated as well. The other subtables are assumed to be
// identical for all tables, so only the subtable of the first table is used.
//
// The ConcatTable maintains its own keyword set, which is initially a copy
// of the keyword set of the first table. It replaces the keywords of the
// subtables to be concatenated.
// The keyword set is not persistent. One can add or change keywords, but
// these changes are not kept when the ConcatTable object is made persistent.
// </synopsis>
 
// <motivation>
// Sometimes a very large MeasurementSet is split into multiple smaller ones
// using the time axis. Using ConcatTable they can still be viewed as a
// single MS. The SYSCAL subtable is split in time as well, thus it has
// to be possible to concatenate that one as well.
// <note>An MS split in subband could be concatenated as well provided that
// at least the first part contains the full SPECTRAL_WINDOW subtable and
// that unique SPWids are used.
// </note>
// </motivation>
 
// <todo asof="$DATE:$">
// # A List of bugs, limitations, extensions or planned refinements.
//   <li> Maybe not allocating the row number vector for a projection.
//          This saves space and time, but each rownr conversion will
//          take a bit more time because it has to test if there is a vector.
//   <li> Maybe maintain a Vector<String> telling on which columns
//          the table is ordered. This may speed up selection, but
//          it is hard to check if the order is changed by a put.
//   <li> Allow to remove a row or column from the ConcatTable
//   <li> Allow to rename a column in the ConcatTable
//   <li> Maybe implement doSort one time for a more efficient sort.
//          (now everything is handled by BaseTable).
// </todo>
 
class ConcatTable : public BaseTable {
 public:
  // Create a virtual table as the concatenation of the given tables.
  // It checks if the table descriptions of the tables are the same.
  // Subtables with the given names will be concatenated as well.
  // It is assumed that the other subtables are the same for all tables,
  // so the ones of the first table are used.
  // <br>The option can be Table::Old or Table::Update.
  // <br>If a non-empty subdirectory name is given, the tables will
  // be moved to that subdirectory when the concatenated table is written
  // (by writeConcatTable).
  // <group>
  ConcatTable(const Block<Table>& tables, const Block<String>& subTables, const String& subDirName);
  ConcatTable(const Block<String>& tableNames, const Block<String>& subTables,
              const String& subDirName, int option, const TableLock& lockOptions,
              const TSMOption& tsmOption);
  // </group>
 
  // Create a concat table out of a file (written by writeConcatTable).
  // The referenced tables will also be opened (if not stored in the cache).
  ConcatTable(AipsIO&, const String& name, rownr_t nrrow, int option, const TableLock& lockOptions,
              const TSMOption& tsmOption);
 
  // The destructor flushes (i.e. writes) the table if it is opened
  // for output and not marked for delete.
  virtual ~ConcatTable();
 
  // Copy constructor is forbidden, because copying a table requires
  // some more knowledge (like table name of result).
  ConcatTable(const ConcatTable&) = delete;
 
  // Assignment is forbidden, because copying a table requires
  // some more knowledge (like table name of result).
  ConcatTable& operator=(const ConcatTable&) = delete;
 
  // Get the names of the tables this table consists of.
  virtual void getPartNames(Block<String>& names, Bool recursive) const;
 
  // Return the layout of a table (i.e. description and #rows).
  // This function has the advantage that only the minimal amount of
  // information required is read from the table, thus it is much
  // faster than a normal table open.
  // <br> The number of rows is returned. The description of the table
  // is stored in desc (its contents will be overwritten).
  static void getLayout(TableDesc& desc, AipsIO& ios);
 
  // Try to reopen the table (the underlying ones) for read/write access.
  // An exception is thrown if the table is not writable.
  // Nothing is done if the table is already open for read/write.
  virtual void reopenRW();
 
  // Is the table stored in big or little endian format?
  // It returns the endianness of the first underlying table.
  virtual Bool asBigEndian() const;
 
  // Get the storage option used for the table.
  // It returns the storage option of the first underlying table.
  virtual const StorageOption& storageOption() const;
 
  // Is the table in use (i.e. open) in another process?
  // It always returns False.
  virtual Bool isMultiUsed(Bool checkSubTable) const;
 
  // Get the locking info.
  // All underlying tables have the same lock option.
  virtual const TableLock& lockOptions() const;
 
  // Merge the given lock info with the existing one.
  virtual void mergeLock(const TableLock& lockOptions);
 
  // Has this process the read or write lock, thus can the table
  // be read or written safely?
  virtual Bool hasLock(FileLocker::LockType) const;
 
  // Try to lock the table for read or write access.
  virtual Bool lock(FileLocker::LockType, uInt nattempts);
 
  // Unlock the table. This will also synchronize the table data,
  // thus force the data to be written to disk.
  virtual void unlock();
 
  // Flush the table, i.e. write it to disk.
  // Nothing will be done if the table is not writable.
  // A flush can be executed at any time.
  // When a table is marked for delete, the destructor will remove
  // files written by intermediate flushes.
  // Note that if necessary the destructor will do an implicit flush,
  // unless it is executed due to an exception.
  virtual void flush(Bool fsync, Bool recursive);
 
  // Resync the Table object with the table files.
  virtual void resync();
 
  // Get the modify counter.
  virtual uInt getModifyCounter() const;
 
  // Test if all underlying tables are opened as writable.
  virtual Bool isWritable() const;
 
  // Read a concat table from a file.
  // The underlying tables will be opened (if not stored in the cache).
  void getConcat(AipsIO&, int option, const TableLock& lockOptions, const TSMOption& tsmOption);
 
  // This is doing a shallow copy.
  // It gives an error if the ConcatTable has not been stored yet.
  virtual void copy(const String& newName, int tableOption) const;
 
  // Copy the table and all its subtables.
  // It copies the contents of each row to get a real copy.
  virtual void deepCopy(const String& newName, const Record& dataManagerInfo, const StorageOption&,
                        int tableOption, Bool, int endianFormat, Bool noRows) const;
 
  // It returns the type of the parent table.
  virtual int tableType() const;
 
  // Get the actual table description.
  virtual TableDesc actualTableDesc() const;
 
  // Get the data manager info (of the first underlying table).
  virtual Record dataManagerInfo() const;
 
  // Get readonly access to the table keyword set.
  virtual TableRecord& keywordSet();
 
  // Get read/write access to the table keyword set.
  // This requires that the table is locked (or it gets locked
  // when using AutoLocking mode).
  virtual TableRecord& rwKeywordSet();
 
  // Get a column object using its index.
  virtual BaseColumn* getColumn(uInt columnIndex) const;
 
  // Get a column object using its name.
  virtual BaseColumn* getColumn(const String& columnName) const;
 
  // Test if it is possible to remove a row from this table (no).
  virtual Bool canRemoveRow() const;
 
  // Remove the given row.
  virtual void removeRow(rownr_t rownr);
 
  // Test if columns can be removed (no).
  virtual Bool canRemoveColumn(const Vector<String>& columnNames) const;
 
  // Add one or more columns to the table.
  // The column is added to the parent tables if told so and if not existing.
  // <group>
  virtual void addColumn(const ColumnDesc& columnDesc, Bool addToParent);
  virtual void addColumn(const ColumnDesc& columnDesc, const String& dataManager, Bool byName,
                         Bool addToParent);
  virtual void addColumn(const ColumnDesc& columnDesc, const DataManager& dataManager,
                         Bool addToParent);
  virtual void addColumn(const TableDesc& tableDesc, const DataManager& dataManager,
                         Bool addToParent);
  // </group>
 
  // Remove a column.
  virtual void removeColumn(const Vector<String>& columnNames);
 
  // Test if a column can be renamed (no).
  virtual Bool canRenameColumn(const String& columnName) const;
 
  // Rename a column.
  virtual void renameColumn(const String& newName, const String& oldName);
 
  // Rename a hypercolumn.
  virtual void renameHypercolumn(const String& newName, const String& oldName);
 
  // Find the data manager with the given name or for the given column.
  virtual DataManager* findDataManager(const String& name, Bool byColumn) const;
 
  // Get the rows object.
  const ConcatRows& rows() const { return rows_p; }
 
  // Get the column objects in the referenced tables.
  Block<BaseColumn*> getRefColumns(const String& columnName);
 
 private:
  // Show the extra table structure info (names of used tables).
  void showStructureExtra(std::ostream&) const;
 
  // Open all tables in the required way.
  void openTables(const Block<String>& tableNames, int option, const TableLock& lockOptions,
                  const TSMOption& tsmOption);
 
  // Initialize.
  // It checks if the descriptions of all tables are equal.
  // It creates the keyword setfor which it concatenates subtables as needed.
  void initialize();
 
  // Setup the main parts of the object.
  // <br>First create the name map (mapping column name in ConcatTable to
  // the column in the original table).
  // If the BaseTable is a ConcatTable, use its name map.
  // Otherwise create the initial name map from the table description.
  // A rename might change the map.
  // <br>Create the ConcatColumn objects.
  // <br>Create the initial TableInfo as a copy of the original BaseTable.
  void setup(BaseTable* btp, const Vector<String>& columnNames);
 
  // Add lines containing the concatenated tables to the info.
  void addInfo();
 
  // Create the ConcatColumn objects for all columns in the description.
  void makeConcatCol();
 
  // Handle the subtales that have to be concatenated.
  void handleSubTables();
 
  // Write a reference table.
  void writeConcatTable(Bool fsync);
 
  // Check if the column can be added, thus does not exist yet.
  void checkAddColumn(const String& name, Bool addToParent);
 
  // Add a column, with internal bookeeping (columns map).
  void addConcatCol(const ColumnDesc& cd);
  // Add multiple columns, with internal bookeeping (columns map).
  void addConcatCol(const TableDesc& tdesc);
 
  // # Data members
  Block<String> subTableNames_p;
  String subDirName_p;
  Block<Table> tables_p;                     // # Tables forming the concat
  std::map<String, ConcatColumn*> colMap_p;  // # map name to column
  TableRecord keywordSet_p;
  Bool changed_p;  // # True = changed since last write
  ConcatRows rows_p;
};
 
}  // namespace casacore
 
#endif