SQL中删除表内的重复数据
表是用于保存修改的历史记录的， 表的结构如下：
TABLE [dbo].[Upd_Record](
[ID] [decimal](18, 0) IDENTITY(1,1) NOT NULL, --标识列
[Table_nm] [varchar](50) NOT NULL,--外联表名
[Table_Cell] [varchar](50) NOT NULL,--单元格名称
[Row_ID] [decimal](18, 0) NOT NULL,--外联主键
[Cell_Value] [decimal](18, 4) NULL,--值 ，主要是这个重复
[upd_date] [datetime] NOT NULL,--修改日期
[upd_id] [decimal](18, 0) NULL--修改人
数据例子如下：
id Table_nm Table_Cell Row_ID Cell_Value
1bxsphz_subclf327564.00002012-03-23 14:34:46.0004195
2bxsphz_subbxywf327495.00002012-03-23 14:34:46.0004195
3bxsphz_subtmoney3270.00002012-03-23 14:34:46.0004195
4bxsphz_subclf327564.00002012-03-25 14:21:35.0002121
5bxsphz_subbxywf327495.00002012-03-25 14:21:35.0002121
6bxsphz_subtmoney3270.00002012-03-25 14:21:35.0002121
需求如下：
如果有修改的数据，保留；
如果2次修改没有改变过数据， 则只保留最早的，如上面的数据 应保留123，删除456；
由于修改日期基本上是越来越大的，所以按照ID大小来删除应该就可以了， 于是LZ写了如下的 sql语句：
select * from Upd_Record
where ID not in(
select g.ID
from Upd_Record g
where exists (
select f.ID
from Upd_Record f
where f.ID < g.ID and not exists (
select Table_nm, Table_Cell, Row_ID, Cell_Value
from Upd_Record
where ID = f.ID or ID = g.ID
group by Table_nm, Table_Cell, Row_ID, Cell_Value
having count(*) =1)))
go
用临时表测试的时候，感觉这个应该是可行的，但是复制到数据表中， 由于数据有 45W多条，卡住了10来分钟仍然在查询中，无法得到结果。
因此特来求教， 有没有一些好的办法来处理 删除重复的数据这样的需求？尤其是大批量的时候
[解决办法]
删除重复记录有大小关系时,保留大或小其中一个记录--> --> (Roy)生成測試數據if not object_id('Tempdb..#T') is null drop table #TGoCreate table #T([ID] int,[Name] nvarchar(1),[Memo] nvarchar(2))Insert #Tselect 1,N'A',N'A1' union allselect 2,N'A',N'A2' union allselect 3,N'A',N'A3' union allselect 4,N'B',N'B1' union allselect 5,N'B',N'B2'Go--I、Name相同ID最小的记录(推荐用1,2,3),保留最小一条方法1:delete a from #T a where exists(select 1 from #T where Name=a.Name and ID<a.ID)方法2:delete a from #T a left join (select min(ID)ID,Name from #T group by Name) b on a.Name=b.Name and a.ID=b.ID where b.Id is null方法3:delete a from #T a where ID not in (select min(ID) from #T where Name=a.Name)方法4(注:ID为唯一时可用):delete a from #T a where ID not in(select min(ID)from #T group by Name)方法5:delete a from #T a where (select count(1) from #T where Name=a.Name and ID<a.ID)>0方法6:delete a from #T a where ID<>(select top 1 ID from #T where Name=a.name order by ID)方法7:delete a from #T a where ID>any(select ID from #T where Name=a.Name)select * from #T生成结果:/*ID Name Memo----------- ---- ----1 A A14 B B1(2 行受影响)*/--II、Name相同ID保留最大的一条记录:方法1:delete a from #T a where exists(select 1 from #T where Name=a.Name and ID>a.ID)方法2:delete a from #T a left join (select max(ID)ID,Name from #T group by Name) b on a.Name=b.Name and a.ID=b.ID where b.Id is null方法3:delete a from #T a where ID not in (select max(ID) from #T where Name=a.Name)方法4(注:ID为唯一时可用):delete a from #T a where ID not in(select max(ID)from #T group by Name)方法5:delete a from #T a where (select count(1) from #T where Name=a.Name and ID>a.ID)>0方法6:delete a from #T a where ID<>(select top 1 ID from #T where Name=a.name order by ID desc)方法7:delete a from #T a where ID<any(select ID from #T where Name=a.Name)select * from #T/*ID Name Memo----------- ---- ----3 A A35 B B2(2 行受影响)*/--3、删除重复记录没有大小关系时，处理重复值--> --> (Roy)生成測試數據 if not object_id('Tempdb..#T') is null drop table #TGoCreate table #T([Num] int,[Name] nvarchar(1))Insert #Tselect 1,N'A' union allselect 1,N'A' union allselect 1,N'A' union allselect 2,N'B' union allselect 2,N'B'Go方法1:if object_id('Tempdb..#') is not null drop table #Select distinct * into # from #T--排除重复记录结果集生成临时表#truncate table #T--清空表insert #T select * from # --把临时表#插入到表#T中--查看结果select * from #T/*Num Name----------- ----1 A2 B(2 行受影响)*/--重新执行测试数据后用方法2方法2:alter table #T add ID int identity--新增标识列godelete a from #T a where exists(select 1 from #T where Num=a.Num and Name=a.Name and ID>a.ID)--只保留一条记录goalter table #T drop column ID--删除标识列--查看结果select * from #T/*Num Name----------- ----1 A2 B(2 行受影响)*/--重新执行测试数据后用方法3方法3:declare Roy_Cursor cursor local forselect count(1)-1,Num,Name from #T group by Num,Name having count(1)>1declare @con int,@Num int,@Name nvarchar(1)open Roy_Cursorfetch next from Roy_Cursor into @con,@Num,@Namewhile @@Fetch_status=0begin set rowcount @con; delete #T where Num=@Num and Name=@Name set rowcount 0; fetch next from Roy_Cursor into @con,@Num,@Nameendclose Roy_Cursordeallocate Roy_Cursor--查看结果select * from #T/*Num Name----------- ----1 A2 B(2 行受影响)*/方法太多了，随便选一种吧
[解决办法]
--> 测试数据：[test]if object_id('[test]') is not null drop table [test]create table [test]([id] int,[Table_nm] varchar(10),[Table_Cell] varchar(6),[Row_ID] int,[Cell_Value] numeric(7,4),[C6] datetime,[C7] int)insert [test]select 1,'bxsphz_sub','clf',327,564.0000,'2012-03-23 14:34:46.000',4195 union allselect 2,'bxsphz_sub','bxywf',327,495.0000,'2012-03-23 14:34:46.000',4195 union allselect 3,'bxsphz_sub','tmoney',327,0.0000,'2012-03-23 14:34:46.000',4195 union allselect 4,'bxsphz_sub','clf',327,564.0000,'2012-03-25 14:21:35.000',2121 union allselect 5,'bxsphz_sub','bxywf',327,495.0000,'2012-03-25 14:21:35.000',2121 union allselect 6,'bxsphz_sub','tmoney',327,0.0000,'2012-03-25 14:21:35.000',2121with tas(select px=ROW_NUMBER()over(partition by [Table_Cell],[Row_ID] order by id),* from test)delete from testwhere id in(select id from t where px>=2)select * from test/*id Table_nm Table_Cell Row_ID Cell_Value C6 C71 bxsphz_sub clf 327 564.0000 2012-03-23 14:34:46.000 41952 bxsphz_sub bxywf 327 495.0000 2012-03-23 14:34:46.000 41953 bxsphz_sub tmoney 327 0.0000 2012-03-23 14:34:46.000 4195*/--我选了用row_number函数实现的，这个效率可能会高点
[解决办法]
select min(b.id)as id,a.* from (select Table_nm, Table_Cell, Row_ID, Cell_Value from Upd_Record group by Table_nm, Table_Cell, Row_ID, Cell_Value)ajoin Upd_Record B on a. Table_nm=b.Table_nm and a.Table_Cell=b.Table_Cell and a.row_id=b.row_idand a.Cell_Value=b.Cell_Valuegroup by a.Table_nm, a.Table_Cell, a.Row_ID, a.Cell_Value