protected static String getTableNameForHCat(TableDesc table, String uuid) { String tableName = (table.isView()) ? table.getMaterializedName(uuid) : table.getName(); String database = (table.isView()) ? KylinConfig.getInstanceFromEnv().getHiveDatabaseForIntermediateTable() : table.getDatabase(); return String.format(Locale.ROOT, "%s.%s", database, tableName).toUpperCase(Locale.ROOT); }
private void generateLoadDataDDL(Set<TableDesc> tables, PrintWriter out) { for (TableDesc t : tables) { if (t.isView()) { out.print("-- " + t.getIdentity() + " is view \n"); continue; } out.print("LOAD DATA LOCAL INPATH '" + t.getIdentity() + ".csv' OVERWRITE INTO TABLE " + normHiveIdentifier(t.getIdentity()) + ";\n"); } }
private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) { for (TableDesc t : tables) { if (t.isView()) continue; out.print("DROP TABLE IF EXISTS " + normHiveIdentifier(t.getIdentity()) + ";\n"); out.print("CREATE TABLE " + normHiveIdentifier(t.getIdentity()) + "(" + "\n"); for (int i = 0; i < t.getColumns().length; i++) { ColumnDesc col = t.getColumns()[i]; out.print(" "); if (i > 0) { out.print(","); } out.print(normHiveIdentifier(col.getName()) + " " + hiveType(col.getType()) + "\n"); } out.print(")" + "\n"); out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n"); out.print("STORED AS TEXTFILE" + ";\n"); out.print("\n"); } }
private static void rebuild(String table, String overwriteUUID, String project) throws IOException { KylinConfig conf = KylinConfig.getInstanceFromEnv(); TableMetadataManager metaMgr = TableMetadataManager.getInstance(conf); SnapshotManager snapshotMgr = SnapshotManager.getInstance(conf); TableDesc tableDesc = metaMgr.getTableDesc(table, project); if (tableDesc == null) throw new IllegalArgumentException("Not table found by " + table); if (tableDesc.isView()) throw new IllegalArgumentException("Build snapshot of hive view \'" + table + "\' not supported."); SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceManager.createReadableTable(tableDesc, null), tableDesc, overwriteUUID); System.out.println("resource path updated: " + snapshot.getResourcePath()); } }
public TableGenConfig(TableDesc table, ModelDataGenerator modelGen) throws IOException { String dataGen = table.getDataGen(); if (dataGen == null && modelGen.existsInStore(table) == false) { dataGen = ""; } if (dataGen == null || "no".equals(dataGen) || "false".equals(dataGen) || "skip".equals(dataGen)) return; if (table.isView()) return; needGen = true; Map<String, String> config = Util.parseEqualCommaPairs(dataGen, "rows"); // config.rows is either a multiplier (0,1] or an absolute row number rows = Util.parseDouble(config, "rows", modelGen.getModle().isFactTable(table.getIdentity()) ? 1.0 : 20); }
@Override public IReadableTable createReadableTable(TableDesc tableDesc, String uuid) { // hive view must have been materialized already // ref HiveMRInput.createLookupHiveViewMaterializationStep() if (tableDesc.isView()) { KylinConfig config = KylinConfig.getInstanceFromEnv(); String tableName = tableDesc.getMaterializedName(uuid); tableDesc = new TableDesc(); tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable()); tableDesc.setName(tableName); } return new HiveTable(tableDesc); }
public CubeInstance rebuildLookupSnapshot(CubeInstance cube, String segmentName, String lookupTable) throws IOException { aclEvaluate.checkProjectOperationPermission(cube); Message msg = MsgPicker.getMsg(); TableDesc tableDesc = getTableManager().getTableDesc(lookupTable, cube.getProject()); if (tableDesc.isView()) { throw new BadRequestException( String.format(Locale.ROOT, msg.getREBUILD_SNAPSHOT_OF_VIEW(), tableDesc.getName())); } CubeSegment seg = cube.getSegment(segmentName, SegmentStatusEnum.READY); getCubeManager().buildSnapshotTable(seg, lookupTable, null); return cube; }
public JobInstance submitLookupSnapshotJob(CubeInstance cube, String lookupTable, List<String> segmentIDs, String submitter) throws IOException { Message msg = MsgPicker.getMsg(); TableDesc tableDesc = getTableManager().getTableDesc(lookupTable, cube.getProject()); if (tableDesc.isView()) { throw new BadRequestException( String.format(Locale.ROOT, msg.getREBUILD_SNAPSHOT_OF_VIEW(), tableDesc.getName())); } LookupSnapshotBuildJob job = new LookupSnapshotJobBuilder(cube, lookupTable, segmentIDs, submitter).build(); getExecutableManager().addJob(job); JobInstance jobInstance = getLookupSnapshotBuildJobInstance(job); return jobInstance; }
for (JoinTableDesc lookupDesc : flatDesc.getDataModel().getJoinTables()) { TableDesc tableDesc = metadataManager.getTableDesc(lookupDesc.getTable(), prj); if (lookupDesc.getKind() == DataModelDesc.TableKind.LOOKUP && tableDesc.isView()) { lookupViewsTables.add(tableDesc); for (TableDesc lookUpTableDesc : lookupViewsTables) { String identity = lookUpTableDesc.getIdentityQuoted("`"); if (lookUpTableDesc.isView()) { String intermediate = lookUpTableDesc.getMaterializedName(uuid); String materializeViewHql = materializeViewHql(intermediate, identity, jobWorkingDir);
Set<String> TABLE_NAMES = new HashSet<String>(); for (TableRef tr : tables) { if (!tr.getTableDesc().isView()) { String tableName = tr.getTableName(); String schema = tr.getTableDesc().getDatabase();
protected static String getTableNameForHCat(TableDesc table, String uuid) { String tableName = (table.isView()) ? table.getMaterializedName(uuid) : table.getName(); String database = (table.isView()) ? KylinConfig.getInstanceFromEnv().getHiveDatabaseForIntermediateTable() : table.getDatabase(); return String.format(Locale.ROOT, "%s.%s", database, tableName).toUpperCase(Locale.ROOT); }
private void generateLoadDataDDL(Set<TableDesc> tables, PrintWriter out) { for (TableDesc t : tables) { if (t.isView()) { out.print("-- " + t.getIdentity() + " is view \n"); continue; } out.print("LOAD DATA LOCAL INPATH '" + t.getIdentity() + ".csv' OVERWRITE INTO TABLE " + normHiveIdentifier(t.getIdentity()) + ";\n"); } }
private static void rebuild(String table, String overwriteUUID, String project) throws IOException { KylinConfig conf = KylinConfig.getInstanceFromEnv(); TableMetadataManager metaMgr = TableMetadataManager.getInstance(conf); SnapshotManager snapshotMgr = SnapshotManager.getInstance(conf); TableDesc tableDesc = metaMgr.getTableDesc(table, project); if (tableDesc == null) throw new IllegalArgumentException("Not table found by " + table); if (tableDesc.isView()) throw new IllegalArgumentException("Build snapshot of hive view \'" + table + "\' not supported."); SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceManager.createReadableTable(tableDesc, null), tableDesc, overwriteUUID); System.out.println("resource path updated: " + snapshot.getResourcePath()); } }
private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) { for (TableDesc t : tables) { if (t.isView()) continue; out.print("DROP TABLE IF EXISTS " + normHiveIdentifier(t.getIdentity()) + ";\n"); out.print("CREATE TABLE " + normHiveIdentifier(t.getIdentity()) + "(" + "\n"); for (int i = 0; i < t.getColumns().length; i++) { ColumnDesc col = t.getColumns()[i]; out.print(" "); if (i > 0) { out.print(","); } out.print(normHiveIdentifier(col.getName()) + " " + hiveType(col.getType()) + "\n"); } out.print(")" + "\n"); out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n"); out.print("STORED AS TEXTFILE" + ";\n"); out.print("\n"); } }
public TableGenConfig(TableDesc table, ModelDataGenerator modelGen) throws IOException { String dataGen = table.getDataGen(); if (dataGen == null && modelGen.existsInStore(table) == false) { dataGen = ""; } if (dataGen == null || "no".equals(dataGen) || "false".equals(dataGen) || "skip".equals(dataGen)) return; if (table.isView()) return; needGen = true; Map<String, String> config = Util.parseEqualCommaPairs(dataGen, "rows"); // config.rows is either a multiplier (0,1] or an absolute row number rows = Util.parseDouble(config, "rows", modelGen.getModle().isFactTable(table.getIdentity()) ? 1.0 : 20); }
@Override public IReadableTable createReadableTable(TableDesc tableDesc, String uuid) { // hive view must have been materialized already // ref HiveMRInput.createLookupHiveViewMaterializationStep() if (tableDesc.isView()) { KylinConfig config = KylinConfig.getInstanceFromEnv(); String tableName = tableDesc.getMaterializedName(uuid); tableDesc = new TableDesc(); tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable()); tableDesc.setName(tableName); } return new HiveTable(tableDesc); }
for (JoinTableDesc lookupDesc : flatDesc.getDataModel().getJoinTables()) { TableDesc tableDesc = metadataManager.getTableDesc(lookupDesc.getTable(), prj); if (lookupDesc.getKind() == DataModelDesc.TableKind.LOOKUP && tableDesc.isView()) { lookupViewsTables.add(tableDesc); for (TableDesc lookUpTableDesc : lookupViewsTables) { String identity = lookUpTableDesc.getIdentityQuoted("`"); if (lookUpTableDesc.isView()) { String intermediate = lookUpTableDesc.getMaterializedName(uuid); String materializeViewHql = materializeViewHql(intermediate, identity, jobWorkingDir);