insert into some_staging_table using Entity Framework.
-- Single insert into main table (this could be a tiny stored proc call)insert into some_main_already_large_table (columns...)select (columns...) from some_staging_tabletruncate table some_staging_table
public void AddAndSave<TEntity>(List<TEntity> entities) where TEntity : class {
const int CommitCount = 1000; //set your own best performance number hereint currentCount = 0;
while (currentCount < entities.Count()){//make sure it don't commit more than the entities you haveint commitCount = CommitCount;if ((entities.Count - currentCount) < commitCount)commitCount = entities.Count - currentCount;
//e.g. Add entities [ i = 0 to 999, 1000 to 1999, ... , n to n+999... ] to conextfor (int i = currentCount; i < (currentCount + commitCount); i++)_context.Entry(entities[i]).State = System.Data.EntityState.Added;//same as calling _context.Set<TEntity>().Add(entities[i]);
//commit entities[n to n+999] to database_context.SaveChanges();
//detach all entities in the context that committed to database//so it won't overload the contextfor (int i = currentCount; i < (currentCount + commitCount); i++)_context.Entry(entities[i]).State = System.Data.EntityState.Detached;
currentCount += commitCount;} }
var bulk = new BulkOperations();var books = GetBooks();
using (TransactionScope trans = new TransactionScope()){using (SqlConnection conn = new SqlConnection(ConfigurationManager.ConnectionStrings["SqlBulkToolsTest"].ConnectionString)){bulk.Setup<Book>().ForCollection(books).WithTable("Books").AddAllColumns().BulkInsert().Commit(conn);}
trans.Complete();}
public static PostgreSQLCopyHelper<T> CreateHelper<T>(string schemaName, string tableName){var helper = new PostgreSQLCopyHelper<T>("dbo", "\"" + tableName + "\"");var properties = typeof(T).GetProperties();foreach(var prop in properties){var type = prop.PropertyType;if (Attribute.IsDefined(prop, typeof(KeyAttribute)) || Attribute.IsDefined(prop, typeof(ForeignKeyAttribute)))continue;switch (type){case Type intType when intType == typeof(int) || intType == typeof(int?):{helper = helper.MapInteger("\"" + prop.Name + "\"", x => (int?)typeof(T).GetProperty(prop.Name).GetValue(x, null));break;}case Type stringType when stringType == typeof(string):{helper = helper.MapText("\"" + prop.Name + "\"", x => (string)typeof(T).GetProperty(prop.Name).GetValue(x, null));break;}case Type dateType when dateType == typeof(DateTime) || dateType == typeof(DateTime?):{helper = helper.MapTimeStamp("\"" + prop.Name + "\"", x => (DateTime?)typeof(T).GetProperty(prop.Name).GetValue(x, null));break;}case Type decimalType when decimalType == typeof(decimal) || decimalType == typeof(decimal?):{helper = helper.MapMoney("\"" + prop.Name + "\"", x => (decimal?)typeof(T).GetProperty(prop.Name).GetValue(x, null));break;}case Type doubleType when doubleType == typeof(double) || doubleType == typeof(double?):{helper = helper.MapDouble("\"" + prop.Name + "\"", x => (double?)typeof(T).GetProperty(prop.Name).GetValue(x, null));break;}case Type floatType when floatType == typeof(float) || floatType == typeof(float?):{helper = helper.MapReal("\"" + prop.Name + "\"", x => (float?)typeof(T).GetProperty(prop.Name).GetValue(x, null));break;}case Type guidType when guidType == typeof(Guid):{helper = helper.MapUUID("\"" + prop.Name + "\"", x => (Guid)typeof(T).GetProperty(prop.Name).GetValue(x, null));break;}}}return helper;}
我使用它的方式如下(我有实体名为承接):
var undertakingHelper = BulkMapper.CreateHelper<Model.Undertaking>("dbo", nameof(Model.Undertaking));undertakingHelper.SaveAll(transaction.UnderlyingTransaction.Connection as Npgsql.NpgsqlConnection, undertakingsToAdd));
// at some point in my calling code, I will call:var myDataTable = CreateMyDataTable();myDataTable.Rows.Add(Guid.NewGuid,tableHeaderId,theName,theValue); // e.g. - need this call for each row to insert
var efConnectionString = ConfigurationManager.ConnectionStrings["MyWebConfigEfConnection"].ConnectionString;var efConnectionStringBuilder = new EntityConnectionStringBuilder(efConnectionString);var connectionString = efConnectionStringBuilder.ProviderConnectionString;BulkInsert(connectionString, myDataTable);
private DataTable CreateMyDataTable(){var myDataTable = new DataTable { TableName = "MyTable"};// this table has an identity column - don't need to specify thatmyDataTable.Columns.Add("MyTableRecordGuid", typeof(Guid));myDataTable.Columns.Add("MyTableHeaderId", typeof(int));myDataTable.Columns.Add("ColumnName", typeof(string));myDataTable.Columns.Add("ColumnValue", typeof(string));return myDataTable;}
private void BulkInsert(string connectionString, DataTable dataTable){using (var connection = new SqlConnection(connectionString)){connection.Open();SqlTransaction transaction = null;try{transaction = connection.BeginTransaction();
using (var sqlBulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.TableLock, transaction)){sqlBulkCopy.DestinationTableName = dataTable.TableName;foreach (DataColumn column in dataTable.Columns) {sqlBulkCopy.ColumnMappings.Add(column.ColumnName, column.ColumnName);}
sqlBulkCopy.WriteToServer(dataTable);}transaction.Commit();}catch (Exception){transaction?.Rollback();throw;}}}
context.ChangeTracker.AutoDetectChangesEnabled = false;foreach (IRecord record in records) {//Add records to your database}context.ChangeTracker.DetectChanges();context.SaveChanges();context.ChangeTracker.AutoDetectChangesEnabled = true; //do not forget to re-enable
abstract class SomeRepository {
protected MyDbContext myDbContextRef;
public void ImportData<TChild, TFather>(List<TChild> entities, TFather entityFather)where TChild : class, IEntityChildwhere TFather : class, IEntityFather{
using (var scope = MyDbContext.CreateTransactionScope()){
MyDbContext context = null;try{context = new MyDbContext(myDbContextRef.ConnectionString);
context.Configuration.AutoDetectChangesEnabled = false;
entityFather.BulkInsertResult = false;var fileEntity = context.Set<TFather>().Add(entityFather);context.SaveChanges();
int count = 0;
//avoids an issue with recreating context: EF duplicates the first commit block of data at the end of transaction!!context = MyDbContext.AddToContext<TChild>(context, null, 0, 1, true);
foreach (var entityToInsert in entities){++count;entityToInsert.EntityFatherRefId = fileEntity.Id;context = MyDbContext.AddToContext<TChild>(context, entityToInsert, count, 100, true);}
entityFather.BulkInsertResult = true;context.Set<TFather>().Add(fileEntity);context.Entry<TFather>(fileEntity).State = EntityState.Modified;
context.SaveChanges();}finally{if (context != null)context.Dispose();}
scope.Complete();}
}
}
仅用于示例目的的接口:
public interface IEntityChild {
//some properties ...
int EntityFatherRefId { get; set; }
}
public interface IEntityFather {
int Id { get; set; }bool BulkInsertResult { get; set; }}
我将解决方案的各种元素实现为静态方法的数据库上下文:
public class MyDbContext : DbContext{
public string ConnectionString { get; set; }
public MyDbContext(string nameOrConnectionString): base(nameOrConnectionString){Database.SetInitializer<MyDbContext>(null);ConnectionString = Database.Connection.ConnectionString;}
/// <summary>/// Creates a TransactionScope raising timeout transaction to 30 minutes/// </summary>/// <param name="_isolationLevel"></param>/// <param name="timeout"></param>/// <remarks>/// It is possible to set isolation-level and timeout to different values. Pay close attention managing these 2 transactions working parameters./// <para>Default TransactionScope values for isolation-level and timeout are the following:</para>/// <para>Default isolation-level is "Serializable"</para>/// <para>Default timeout ranges between 1 minute (default value if not specified a timeout) to max 10 minute (if not changed by code or updating max-timeout machine.config value)</para>/// </remarks>public static TransactionScope CreateTransactionScope(IsolationLevel _isolationLevel = IsolationLevel.Serializable, TimeSpan? timeout = null){SetTransactionManagerField("_cachedMaxTimeout", true);SetTransactionManagerField("_maximumTimeout", timeout ?? TimeSpan.FromMinutes(30));
var transactionOptions = new TransactionOptions();transactionOptions.IsolationLevel = _isolationLevel;transactionOptions.Timeout = TransactionManager.MaximumTimeout;return new TransactionScope(TransactionScopeOption.Required, transactionOptions);}
private static void SetTransactionManagerField(string fieldName, object value){typeof(TransactionManager).GetField(fieldName, BindingFlags.NonPublic | BindingFlags.Static).SetValue(null, value);}
/// <summary>/// Adds a generic entity to a given context allowing commit on large block of data and improving performance to support db bulk-insert operations based on Entity Framework/// </summary>/// <typeparam name="T"></typeparam>/// <param name="context"></param>/// <param name="entity"></param>/// <param name="count"></param>/// <param name="commitCount">defines the block of data size</param>/// <param name="recreateContext"></param>/// <returns></returns>public static MyDbContext AddToContext<T>(MyDbContext context, T entity, int count, int commitCount, bool recreateContext) where T : class{if (entity != null)context.Set<T>().Add(entity);
if (count % commitCount == 0){context.SaveChanges();if (recreateContext){var contextConnectionString = context.ConnectionString;context.Dispose();context = new MyDbContext(contextConnectionString);context.Configuration.AutoDetectChangesEnabled = false;}}
return context;}}