def remove_duplicates(input_list):
  if input_list == []:
    return []
  #sort list from smallest to largest
  input_list=sorted(input_list)
  #initialize ouput list with first element of the       sorted input list
  output_list = [input_list[0]]
  for item in input_list:
    if item >output_list[-1]:
      output_list.append(item)
  return output_list   

减少变量与顺序保存:

假设我们有一个列表:

l = [5, 6, 6, 1, 1, 2, 2, 3, 4]

减少变体(无效):

>>> reduce(lambda r, v: v in r and r or r + [v], l, [])
[5, 6, 1, 2, 3, 4]

5倍快，但更复杂

>>> reduce(lambda r, v: v in r[1] and r or (r[0].append(v) or r[1].add(v)) or r, l, ([], set()))[0]
[5, 6, 1, 2, 3, 4]

解释:

default = (list(), set())
# user list to keep order
# use set to make lookup faster

def reducer(result, item):
    if item not in result[1]:
        result[0].append(item)
        result[1].add(item)
    return result

reduce(reducer, l, default)[0]

如果你不关心顺序，想要一些不同于上面建议的python方式(也就是说，它可以在面试中使用)，那么:

def remove_dup(arr):
    size = len(arr)
    j = 0    # To store index of next unique element
    for i in range(0, size-1):
        # If current element is not equal
        # to next element then store that
        # current element
        if(arr[i] != arr[i+1]):
            arr[j] = arr[i]
            j+=1

    arr[j] = arr[size-1] # Store the last element as whether it is unique or repeated, it hasn't stored previously

    return arr[0:j+1]

if __name__ == '__main__':
    arr = [10, 10, 1, 1, 1, 3, 3, 4, 5, 6, 7, 8, 8, 9]
    print(remove_dup(sorted(arr)))

时间复杂度:O(n)

辅助空间:O(n)

参考:http://www.geeksforgeeks.org/remove-duplicates-sorted-array/

您可以使用以下函数:

def rem_dupes(dup_list): 
    yooneeks = [] 
    for elem in dup_list: 
        if elem not in yooneeks: 
            yooneeks.append(elem) 
    return yooneeks

例子:

my_list = ['this','is','a','list','with','dupicates','in', 'the', 'list']

用法:

rem_dupes(my_list)

[‘这个’,‘是’,‘“,“列表”,“与”,“dupicates”,“在”,“的”)

检查字符串'a'和'b'

clean_list = []
    for ele in raw_list:
        if 'b' in ele or 'a' in ele:
            pass
        else:
            clean_list.append(ele)

我已将各种建议与perfplot进行了比较。事实证明，如果输入数组没有重复的元素，所有方法的速度或多或少都一样快，与输入数据是Python列表还是NumPy数组无关。

如果输入数组很大，但只包含一个唯一元素，则set、dict和np。如果输入数据是一个列表，唯一方法是常量时间的。如果是NumPy数组，np。Unique比其他选项快10倍。

让我有点惊讶的是这些也不是常时间运算。

代码重现图:

import perfplot
import numpy as np
import matplotlib.pyplot as plt


def setup_list(n):
    # return list(np.random.permutation(np.arange(n)))
    return [0] * n


def setup_np_array(n):
    # return np.random.permutation(np.arange(n))
    return np.zeros(n, dtype=int)


def list_set(data):
    return list(set(data))


def numpy_unique(data):
    return np.unique(data)


def list_dict(data):
    return list(dict.fromkeys(data))


b = perfplot.bench(
    setup=[
        setup_list,
        setup_list,
        setup_list,
        setup_np_array,
        setup_np_array,
        setup_np_array,
    ],
    kernels=[list_set, numpy_unique, list_dict, list_set, numpy_unique, list_dict],
    labels=[
        "list(set(lst))",
        "np.unique(lst)",
        "list(dict(lst))",
        "list(set(arr))",
        "np.unique(arr)",
        "list(dict(arr))",
    ],
    n_range=[2 ** k for k in range(23)],
    xlabel="len(array)",
    equality_check=None,
)
# plt.title("input array = [0, 1, 2,..., n]")
plt.title("input array = [0, 0,..., 0]")
b.save("out.png")
b.show()