class DataSet:\n",
" """\n",
" A data set for a machine learning problem. It has the following fields:\n",
"\n",
" d.examples A list of examples. Each one is a list of attribute values.\n",
" d.attrs A list of integers to index into an example, so example[attr]\n",
" gives a value. Normally the same as range(len(d.examples[0])).\n",
" d.attr_names Optional list of mnemonic names for corresponding attrs.\n",
" d.target The attribute that a learning algorithm will try to predict.\n",
" By default the final attribute.\n",
" d.inputs The list of attrs without the target.\n",
" d.values A list of lists: each sublist is the set of possible\n",
" values for the corresponding attribute. If initially None,\n",
" it is computed from the known examples by self.set_problem.\n",
" If not None, an erroneous value raises ValueError.\n",
" d.distance A function from a pair of examples to a non-negative number.\n",
" Should be symmetric, etc. Defaults to mean_boolean_error\n",
" since that can handle any field types.\n",
" d.name Name of the data set (for output display only).\n",
" d.source URL or other source where the data came from.\n",
" d.exclude A list of attribute indexes to exclude from d.inputs. Elements\n",
" of this list can either be integers (attrs) or attr_names.\n",
"\n",
" Normally, you call the constructor and you're done; then you just\n",
" access fields like d.examples and d.target and d.inputs.\n",
" """\n",
"\n",
" def __init__(self, examples=None, attrs=None, attr_names=None, target=-1, inputs=None,\n",
" values=None, distance=mean_boolean_error, name='', source='', exclude=()):\n",
" """\n",
" Accepts any of DataSet's fields. Examples can also be a\n",
" string or file from which to parse examples using parse_csv.\n",
" Optional parameter: exclude, as documented in .set_problem().\n",
" >>> DataSet(examples='1, 2, 3')\n",
" <DataSet(): 1 examples, 3 attributes>\n",
" """\n",
" self.name = name\n",
" self.source = source\n",
" self.values = values\n",
" self.distance = distance\n",
" self.got_values_flag = bool(values)\n",
"\n",
" # initialize .examples from string or list or data directory\n",
" if isinstance(examples, str):\n",
" self.examples = parse_csv(examples)\n",
" elif examples is None:\n",
" self.examples = parse_csv(open_data(name + '.csv').read())\n",
" else:\n",
" self.examples = examples\n",
"\n",
" # attrs are the indices of examples, unless otherwise stated.\n",
" if self.examples is not None and attrs is None:\n",
" attrs = list(range(len(self.examples[0])))\n",
"\n",
" self.attrs = attrs\n",
"\n",
" # initialize .attr_names from string, list, or by default\n",
" if isinstance(attr_names, str):\n",
" self.attr_names = attr_names.split()\n",
" else:\n",
" self.attr_names = attr_names or attrs\n",
" self.set_problem(target, inputs=inputs, exclude=exclude)\n",
"\n",
" def set_problem(self, target, inputs=None, exclude=()):\n",
" """\n",
" Set (or change) the target and/or inputs.\n",
" This way, one DataSet can be used multiple ways. inputs, if specified,\n",
" is a list of attributes, or specify exclude as a list of attributes\n",
" to not use in inputs. Attributes can be -n .. n, or an attr_name.\n",
" Also computes the list of possible values, if that wasn't done yet.\n",
" """\n",
" self.target = self.attr_num(target)\n",
" exclude = list(map(self.attr_num, exclude))\n",
" if inputs:\n",
" self.inputs = remove_all(self.target, inputs)\n",
" else:\n",
" self.inputs = [a for a in self.attrs if a != self.target and a not in exclude]\n",
" if not self.values:\n",
" self.update_values()\n",
" self.check_me()\n",
"\n",
" def check_me(self):\n",
" """Check that my fields make sense."""\n",
" assert len(self.attr_names) == len(self.attrs)\n",
" assert self.target in self.attrs\n",
" assert self.target not in self.inputs\n",
" assert set(self.inputs).issubset(set(self.attrs))\n",
" if self.got_values_flag:\n",
" # only check if values are provided while initializing DataSet\n",
" list(map(self.check_example, self.examples))\n",
"\n",
" def add_example(self, example):\n",
" """Add an example to the list of examples, checking it first."""\n",
" self.check_example(example)\n",
" self.examples.append(example)\n",
"\n",
" def check_example(self, example):\n",
" """Raise ValueError if example has any invalid values."""\n",
" if self.values:\n",
" for a in self.attrs:\n",
" if example[a] not in self.values[a]:\n",
" raise ValueError('Bad value {} for attribute {} in {}'\n",
" .format(example[a], self.attr_names[a], example))\n",
"\n",
" def attr_num(self, attr):\n",
" """Returns the number used for attr, which can be a name, or -n .. n-1."""\n",
" if isinstance(attr, str):\n",
" return self.attr_names.index(attr)\n",
" elif attr < 0:\n",
" return len(self.attrs) + attr\n",
" else:\n",
" return attr\n",
"\n",
" def update_values(self):\n",
" self.values = list(map(unique, zip(*self.examples)))\n",
"\n",
" def sanitize(self, example):\n",
" """Return a copy of example, with non-input attributes replaced by None."""\n",
" return [attr_i if i in self.inputs else None for i, attr_i in enumerate(example)][:-1]\n",
"\n",
" def classes_to_numbers(self, classes=None):\n",
" """Converts class names to numbers."""\n",
" if not classes:\n",
" # if classes were not given, extract them from values\n",
" classes = sorted(self.values[self.target])\n",
" for item in self.examples:\n",
" item[self.target] = classes.index(item[self.target])\n",
"\n",
" def remove_examples(self, value=''):\n",
" """Remove examples that contain given value."""\n",
" self.examples = [x for x in self.examples if value not in x]\n",
" self.update_values()\n",
"\n",
" def split_values_by_classes(self):\n",
" """Split values into buckets according to their class."""\n",
" buckets = defaultdict(lambda: [])\n",
" target_names = self.values[self.target]\n",
"\n",
" for v in self.examples:\n",
" item = [a for a in v if a not in target_names] # remove target from item\n",
" buckets[v[self.target]].append(item) # add item to bucket of its class\n",
"\n",
" return buckets\n",
"\n",
" def find_means_and_deviations(self):\n",
" """\n",
" Finds the means and standard deviations of self.dataset.\n",
" means : a dictionary for each class/target. Holds a list of the means\n",
" of the features for the class.\n",
" deviations: a dictionary for each class/target. Holds a list of the sample\n",
" standard deviations of the features for the class.\n",
" """\n",
" target_names = self.values[self.target]\n",
" feature_numbers = len(self.inputs)\n",
"\n",
" item_buckets = self.split_values_by_classes()\n",
"\n",
" means = defaultdict(lambda: [0] * feature_numbers)\n",
" deviations = defaultdict(lambda: [0] * feature_numbers)\n",
"\n",
" for t in target_names:\n",
" # find all the item feature values for item in class t\n",
" features = [[] for _ in range(feature_numbers)]\n",
" for item in item_buckets[t]:\n",
" for i in range(feature_numbers):\n",
" features[i].append(item[i])\n",
"\n",
" # calculate means and deviations fo the class\n",
" for i in range(feature_numbers):\n",
" means[t][i] = mean(features[i])\n",
" deviations[t][i] = stdev(features[i])\n",
"\n",
" return means, deviations\n",
"\n",
" def __repr__(self):\n",
" return '<DataSet({}): {:d} examples, {:d} attributes>'.format(self.name, len(self.examples), len(self.attrs))\n",
"