diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index d3d73d75..00000000 --- a/.coveragerc +++ /dev/null @@ -1,4 +0,0 @@ -[run] -omit = - urbansim/urbanchoice/pmat.py - urbansim/**/tests/* diff --git a/.travis.yml b/.travis.yml index 15b74bbe..e94498cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,42 +1,28 @@ language: python -sudo: false + python: -- '2.7' -- '3.5' + - '2.7' + - '3.5' + - '3.6' + - '3.7' + - '3.8' + install: -- if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh - -O miniconda.sh; else wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh - -O miniconda.sh; fi -- bash miniconda.sh -b -p $HOME/miniconda -- export PATH="$HOME/miniconda/bin:$PATH" -- hash -r -- conda config --set always_yes yes --set changeps1 no -- conda update -q conda -- conda info -a -- > - conda create -q -n test-environment - python=$TRAVIS_PYTHON_VERSION - cytoolz ipython-notebook jinja2 matplotlib numpy pandas patsy pip scipy - statsmodels pytables pytest pyyaml toolz -- source activate test-environment -- pip install orca osmnet pandana bottle simplejson zbox -- pip install pytest-cov coveralls pycodestyle -- pip install . + - pip install . + - pip install -r requirements-dev.txt + - pip list + - pip show urbansim + before_script: -- git clone https://github.com/udst/sanfran_urbansim.git -- cd sanfran_urbansim; jupyter nbconvert --to python Simulation.ipynb -- cd .. + - git clone https://github.com/udst/sanfran_urbansim.git + - cd sanfran_urbansim; jupyter nbconvert --to python Simulation.ipynb + - cd .. + script: -- pycodestyle urbansim scripts -- py.test --cov urbansim --cov-report term-missing -- cd sanfran_urbansim; python Simulation.py -- cd .. + - pycodestyle --max-line-length=100 urbansim + - py.test --cov urbansim --cov-report term-missing + - cd sanfran_urbansim; python Simulation.py + - cd .. + after_success: -- coveralls -- bin/build_docs.sh -notifications: - slack: - secure: LnbM2nYkvZF7FreNhqG+ExMeIsbbOWuuOi8ewZCCvMEMsYBEc9oE1hi23enpGUMElGzCTKOdCpQ/Z/QNHMycGZaL/wAZm5EPGNM7CZG1FKDrPYOTJ18ipi+k72k3GuqXM3DwGzjcRO6sw7kdy1k/3uVu8QHmkdgijLT6smIh6r4= -env: - global: - - secure: c2olC+qzN1VHJZlznPy7mUtRvAyU7I2C7N3sYkv71Ds+dx5WN8InF6mpwNs/ZKfERbbA9slSVmYjB4j2c+VT8S1J33o6aI4F4hWNr6l7DgBKoPmozvaDx0GQ45hMOeSHSG5OF+DZCSTEXtxQK82uj61sCvZ6dIlDOqRvAHCjQDI= + - coveralls diff --git a/HISTORY.rst b/CHANGELOG.rst similarity index 90% rename from HISTORY.rst rename to CHANGELOG.rst index c374db45..5935ff95 100644 --- a/HISTORY.rst +++ b/CHANGELOG.rst @@ -1,3 +1,13 @@ +v3.2 +==== + +2020/05/05 + +* Improved installation and compatibility +* Support for Pandas 1.0 +* Various improvements and bug fixes +* Note that active development of certain UrbanSim components has moved to stand-alone libraries in UDST: Developer, Choicemodels, UrbanSim Templates + v3.1.1 ====== diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a2f93a4..a580cea6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,41 +1,80 @@ -Contributing to UrbanSim -======================== +Thanks for using UrbanSim! -Style ------ +This is an open source project that's part of the Urban Data Science Toolkit. Development and maintenance is a collaboration between UrbanSim Inc, U.C. Berkeley's Urban Analytics Lab, and other contributors. -- Python code should follow the [PEP 8 Style Guide][pep8]. -- Python docstrings should follow the [NumPy documentation format][numpydoc]. +You can contact Sam Maurer, the lead maintainer, at `maurer@urbansim.com`. -### Imports -Imports should be one per line. -Imports should be grouped into standard library, third-party, -and intra-library imports. `from` import should follow "regular" `imports`. -Within each group the imports should be alphabetized. -Here's an example: +## If you have a problem: -```python -import sys -from glob import glob +- Take a look at the [open issues](https://github.com/UDST/urbansim/issues) and [closed issues](https://github.com/UDST/urbansim/issues?q=is%3Aissue+is%3Aclosed) to see if there's already a related discussion -import numpy as np +- Open a new issue describing the problem -- if possible, include any error messages, the operating system and version of python you're using, and versions of any libraries that may be relevant -import urbansim.urbansim.modelcompile as modelcompile -from urbansim.util import misc -``` -Imports of scientific Python libraries should follow these conventions: +## Feature proposals: -```python -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -import scipy as sp -``` +- Take a look at the [open issues](https://github.com/UDST/urbansim/issues) and [closed issues](https://github.com/UDST/urbansim/issues?q=is%3Aissue+is%3Aclosed) to see if there's already a related discussion +- Post your proposal as a new issue, so we can discuss it (some proposals may not be a good fit for the project) -Thanks! +- Please note that active development of certain UrbanSim components has moved to stand-alone libraries in UDST: Developer, Choicemodels, UrbanSim Templates -[pep8]: http://legacy.python.org/dev/peps/pep-0008/ -[numpydoc]: https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt + +## Contributing code: + +- Create a new branch of `UDST/urbansim`, or fork the repository to your own account + +- Make your changes, following the existing styles for code and inline documentation + +- Add [tests](https://github.com/UDST/urbansim/tree/master/urbansim/tests) if possible! + +- Open a pull request to the `UDST/urbansim` dev branch, including a writeup of your changes -- take a look at some of the closed PR's for examples + +- Current maintainers will review the code, suggest changes, and hopefully merge it! + + +## Updating the documentation: + +- See instructions in `docs/README.md` + + +## Preparing a release: + +- Make a new branch for release prep + +- Update the version number and changelog + - `CHANGELOG.md` + - `setup.py` + - `urbansim/__init__.py` + - `docs/source/index.rst` + +- Make sure all the tests are passing, and check if updates are needed to `README.md` or to the documentation + +- Open a pull request to the master branch to finalize it + +- After merging, tag the release on GitHub and follow the distribution procedures below + + +## Distributing a release on PyPI (for pip installation): + +- Register an account at https://pypi.org, ask one of the current maintainers to add you to the project, and `pip install twine` + +- Check out the copy of the code you'd like to release + +- Run `python setup.py sdist bdist_wheel --universal` + +- This should create a `dist` directory containing two package files -- delete any old ones before the next step + +- Run `twine upload dist/*` -- this will prompt you for your pypi.org credentials + +- Check https://pypi.org/project/urbansim/ for the new version + + +## Distributing a release on Conda Forge (for conda installation): + +- The [conda-forge/urbansim-feedstock](https://github.com/conda-forge/urbansim-feedstock) repository controls the Conda Forge release + +- Conda Forge bots usually detect new releases on PyPI and set in motion the appropriate feedstock updates, which a current maintainer will need to approve and merge + +- Check https://anaconda.org/conda-forge/urbansim for the new version (may take a few minutes for it to appear) diff --git a/LICENSE b/LICENSE.txt similarity index 96% rename from LICENSE rename to LICENSE.txt index da52c471..c6b9d706 100644 --- a/LICENSE +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2016, UrbanSim, Inc. All rights reserved. +Copyright (c) 2020, UrbanSim, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/MANIFEST.in b/MANIFEST.in index cafbd94a..9696a5cd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,5 @@ -include ez_setup.py +include CHANGELOG.rst +include CONTRIBUTING.md +include LICENSE.txt include README.rst +include requirements-dev.txt \ No newline at end of file diff --git a/README.rst b/README.rst index 975d1123..dd50259c 100644 --- a/README.rst +++ b/README.rst @@ -1,105 +1,52 @@ UrbanSim ======== +.. image:: https://img.shields.io/pypi/v/urbansim.svg + :target: https://pypi.python.org/pypi/urbansim/ + :alt: Latest Version + .. image:: https://travis-ci.org/UDST/urbansim.svg?branch=master - :alt: Build Status (Linux) + :alt: Build Status :target: https://travis-ci.org/UDST/urbansim +.. image:: https://coveralls.io/repos/UDST/urbansim/badge.svg?branch=master + :alt: Test Coverage + :target: https://coveralls.io/r/UDST/urbansim?branch=master -.. image:: https://ci.appveyor.com/api/projects/status/0ygo756020jpcrg3?svg=true - :alt: Build Status (Windows) - :target: https://ci.appveyor.com/project/pksohn/urbansim +UrbanSim is a platform for building statistical models of cities and regions. These models help forecast long-range patterns in real estate development, demographics, and related outcomes, under various policy scenarios. +This ``urbansim`` Python library is a core component. It contains tools for statistical estimation and simulation; domain-specific logic about housing markets, household relocation, and other processes; and frameworks and utilities for assembling a model. -.. image:: https://coveralls.io/repos/UDST/urbansim/badge.png?branch=master - :alt: Test Coverage - :target: https://coveralls.io/r/UDST/urbansim?branch=master +How it works +------------ + +Operational UrbanSim models begin with detailed data about a particular region, and then estimate and validate a system of interconnected model components. Full models draw on a number of libraries: not just ``urbansim``, but also `Orca `__ for task orchestration, `Synthpop `__ for population synthesis, `Pandana `__ for network analysis, and so on. Collectively, these make up the `Urban Data Science Toolkit `__ (UDST). + +UrbanSim models are used by public agencies, consultancies, and researchers in dozens of cities around the U.S. and world. The core platform is open source, but many operational models make use of additional cloud-hosted model building and visualization tools provided by `UrbanSim Inc. `__ + +Learn More +---------- + +* `An Introduction to UrbanSim `__ + +* `UrbanSim for San Francisco: An example implementation `__ + +* `UrbanSim Inc. `__ + +Installation +------------ + +* ``pip install urbansim`` + +* ``conda install urbansim --channel conda-forge`` + +Technical documentation +----------------------- + +* `Getting started `__ + +* `Full documentation `__ + +* Other `UDST `__ libraries -New version of UrbanSim, a tool for modeling metropolitan real estate -markets - -.. image:: http://i.imgur.com/4YyN8ob.jpg - :alt: UrbanSim - -`Detailed documentation `__ for -UrbanSim is now available. - -`Click -here `__ -for installation instructions. - -Let us know what you are working on or if you think you have a great use case -by tweeting us at ``@urbansim`` or post on the UrbanSim `forum`_. - -UrbanSim History ----------------- - -UrbanSim (http://www.urbansim.com) is a model system for analyzing -urban development. It is an open source platform that has been -continuously refined and distributed for planning applications around -the world for over 15 years. Part of the evolution of the platform is -the necessary process of re-engineering the code to take advantage of -new developments in computational libraries and infrastructure. We -implemented UrbanSim initially in Java in the late 1990's, and by 2005 -determined that it was time to re-implement it in Python, and created -the Open Platform for Urban Simulation (OPUS) software implementation at -that time. Now, almost a decade later, it is time again to revisit the -implementation to take advantage of an amazing amount of innovation in -the scientific computing community. The new implementation is hosted on -this GitHub site, and maintained by UrbanSim Inc. and a growing -community of contributors. - -New UrbanSim Implementation ---------------------------- - -This new code base is a streamlined complete re-implementation of the -longstanding UrbanSim project aimed at *reducing the complexity* of -using the UrbanSim methodology. Redesigned from the ground up, the new -library is trivial to install, the development process is made -transparent via this GitHub site, and exhaustive documentation has been -created in the hopes of making modeling much more widely accessible to -planners and new modelers. - -We lean heavily on the `PyData `__ community to make -our work easier - Pandas, `IPython `__, and -`statsmodels `__ are ubiquitous in -this work. These Python libraries essentially replace the UrbanSim -Dataset class, tools to read and write from other storage, and some of -the statistical estimation previously implemented by UrbanSim. - -This makes our task easier as we can focus on urban modeling and leave -the infrastructure to the wider Python community. The -`Pandas `__ library is the core of the new -UrbanSim, which is an extremely popular data manipulation library with a -large community providing support and a very helpful -`book `__. - -We have now converted a full set of UrbanSim models to the new -framework, and have running applications for the Paris, Albuquerque, -Denver, Bay Area, and Detroit regions. We have implemented a complete -set of hedonic price models, location choice models, relocation and -transition models, as well as a new real estate development model using -proforma analysis. - -We do strongly recommend that you contact the team at www.urbansim.com about your -project to make sure you can get support when you need it, -and know what you are getting into. For major applied projects, -professional support is highly recommended. - -Reporting bugs and contributing to UrbanSim --------------------------------------------- - -Please report any bugs you encounter via `GitHub Issues `__. - -If you have improvements or new features you would like to see in UrbanSim: - -1. Open a feature request via `GitHub Issues `__. -2. See our code contribution instructions `here `__. -3. Contribute your code from a fork or branch by using a Pull Request and request a review so it can be considered as an addition to the codebase. - -Academic literature -------------------- - -A selection of academic literature on UrbanSim can be found `here `_. - -.. _forum: http://discussion.urbansim.com/ \ No newline at end of file +* Documentation for `UrbanCanvas `__, the UrbanSim cloud platform diff --git a/appveyor.yml b/appveyor.yml index 075cdeca..174546d5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,29 +6,34 @@ environment: MINICONDA: C:\Miniconda - PYTHON_VERSION: 3.5 MINICONDA: C:\Miniconda3 + - PYTHON_VERSION: 3.6 + MINICONDA: C:\Miniconda3 + - PYTHON_VERSION: 3.7 + MINICONDA: C:\Miniconda3 + # - PYTHON_VERSION: 3.8 # doesn't run yet + # MINICONDA: C:\Miniconda3 init: - "ECHO %PYTHON_VERSION% %MINICONDA%" +# Requirements or sub-requirements that don't install well with pip +# are included directly in the conda environment + install: - "set PATH=%MINICONDA%;%MINICONDA%\\Scripts;%PATH%" - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - - conda info -a - - "conda create -q -n test-environment python=%PYTHON_VERSION% cytoolz ipython-notebook jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pytables pytest pyyaml toolz" - - activate test-environment - - conda install -c conda-forge shapely geopandas - - pip install orca osmnet pandana bottle simplejson zbox - - pip install pycodestyle + - conda update conda --yes --quiet + - "conda create --name test-env python=%PYTHON_VERSION% pip geopandas --yes --quiet" + - activate test-env - pip install . + - pip install -r requirements-dev.txt + - pip show urbansim before_test: -# Update sanfran_urbansim clone once python3 branch is merged - - git clone -b python3 https://github.com/udst/sanfran_urbansim.git + - git clone https://github.com/udst/sanfran_urbansim.git - cd sanfran_urbansim & jupyter nbconvert --to python Simulation.ipynb - cd.. test_script: - - pycodestyle urbansim scripts + - pycodestyle --max-line-length=100 urbansim - pytest urbansim - cd sanfran_urbansim & python Simulation.py \ No newline at end of file diff --git a/bin/build_docs.sh b/bin/build_docs.sh deleted file mode 100755 index 9360bb40..00000000 --- a/bin/build_docs.sh +++ /dev/null @@ -1,61 +0,0 @@ -#! /usr/bin/env bash - -# Copied from github.com/sympy/sympy -# -# This file automatically deploys changes to http://udst.github.io/urbansim/. -# This will only happen when building a non-pull request build on the master -# branch of UrbanSim. -# It requires an access token which should be present in .travis.yml file. -# -# Following is the procedure to get the access token: -# -# $ curl -X POST -u -H "Content-Type: application/json" -d\ -# "{\"scopes\":[\"public_repo\"],\"note\":\"token for pushing from travis\"}"\ -# https://api.github.com/authorizations -# -# It'll give you a JSON response having a key called "token". -# -# $ gem install travis -# $ travis encrypt -r sympy/sympy GH_TOKEN= env.global -# -# This will give you an access token("secure"). This helps in creating an -# environment variable named GH_TOKEN while building. -# -# Add this secure code to .travis.yml as described here http://docs.travis-ci.com/user/encryption-keys/ - -# Exit on error -set -e - -ACTUAL_TRAVIS_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER| cut -d'.' -f 2` - -if [ "$TRAVIS_REPO_SLUG" == "UDST/urbansim" ] && \ - [ "$TRAVIS_BRANCH" == "master" ] && \ - [ "$TRAVIS_PULL_REQUEST" == "false" ] && \ - [ "$ACTUAL_TRAVIS_JOB_NUMBER" == "1" ]; then - - echo "Installing dependencies" - conda install --yes --quiet sphinx numpydoc - pip install sphinx_rtd_theme - - echo "Building docs" - cd docs - make clean - make html - - cd ../../ - echo "Setting git attributes" - git config --global user.email "jiffyclub@gmail.com" - git config --global user.name "Matt Davis" - - echo "Cloning repository" - git clone --quiet --single-branch --branch=gh-pages https://${GH_TOKEN}@github.com/UDST/urbansim.git gh-pages > /dev/null 2>&1 - - cd gh-pages - rm -rf * - cp -R ../urbansim/docs/_build/html/* ./ - git add -A . - - git commit -am "Update dev doc after building $TRAVIS_BUILD_NUMBER" - echo "Pushing commit" - git push -fq origin gh-pages > /dev/null 2>&1 -fi diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..3f4a308e --- /dev/null +++ b/docs/README.md @@ -0,0 +1,31 @@ +This folder generates the UrbanSim online documentation, hosted at https://udst.github.io/urbansim/. + +### How it works + +HTML files are generated using [Sphinx](http://sphinx-doc.org) and hosted with GitHub Pages from the `gh-pages` branch of the repository. The online documentation is rendered and updated **manually**. + +### Editing the documentation + +The files in `docs/source`, along with docstrings in the source code, determine what appears in the rendered documentation. Here's a [good tutorial](https://pythonhosted.org/an_example_pypi_project/sphinx.html) for Sphinx. + +### Previewing changes locally + +Install the copy of UrbanSim that the documentation is meant to reflect. Install the documentation tools. + +``` +pip install . +pip install sphinx sphinx_rtd_theme numpydoc +``` + +Build the documentation. There should be status messages and warnings, but no errors. + +``` +cd docs +sphinx-build -b html source build +``` + +The HTML files will show up in `docs/build/`. + +### Uploading changes + +Clone a second copy of the repository and check out the `gh-pages` branch. Copy over the updated HTML files, commit them, and push the changes to GitHub. diff --git a/docs/build/.gitignore b/docs/build/.gitignore new file mode 100644 index 00000000..150f68c8 --- /dev/null +++ b/docs/build/.gitignore @@ -0,0 +1 @@ +*/* diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 986d03ef..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,33 +0,0 @@ -.. urbansim documentation master file, created by - sphinx-quickstart on Thu May 22 12:13:43 2014. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -UrbanSim -======== - -`UrbanSim `_ is a model system for analyzing urban -development. -It is an open source platform that has been continuously refined and -distributed for planning applications around the world for over 15 years. - -Contents --------- - -.. toctree:: - :maxdepth: 2 - - gettingstarted - examples - models/index - developer/index - utils/index - maps/index - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/docs/Makefile b/docs/source/Makefile similarity index 100% rename from docs/Makefile rename to docs/source/Makefile diff --git a/docs/_static/.placeholder b/docs/source/_static/.placeholder similarity index 100% rename from docs/_static/.placeholder rename to docs/source/_static/.placeholder diff --git a/docs/_templates/.placeholder b/docs/source/_templates/.placeholder similarity index 100% rename from docs/_templates/.placeholder rename to docs/source/_templates/.placeholder diff --git a/docs/conf.py b/docs/source/conf.py similarity index 99% rename from docs/conf.py rename to docs/source/conf.py index 6d442eef..ef1a8656 100644 --- a/docs/conf.py +++ b/docs/source/conf.py @@ -50,16 +50,16 @@ # General information about the project. project = u'urbansim' -copyright = u'2017, UrbanSim Inc' +copyright = u'2020, UrbanSim Inc.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '3.1' +version = '3.2' # The full version, including alpha/beta/rc tags. -release = '3.1.1' +release = '3.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/developer/index.rst b/docs/source/developer/index.rst similarity index 100% rename from docs/developer/index.rst rename to docs/source/developer/index.rst diff --git a/docs/examples.rst b/docs/source/examples.rst similarity index 100% rename from docs/examples.rst rename to docs/source/examples.rst diff --git a/docs/gettingstarted.rst b/docs/source/gettingstarted.rst similarity index 81% rename from docs/gettingstarted.rst rename to docs/source/gettingstarted.rst index 0e7ad269..770b65cd 100644 --- a/docs/gettingstarted.rst +++ b/docs/source/gettingstarted.rst @@ -1,120 +1,48 @@ Getting Started =============== -Let us know what you are working on or if you think you have a great use case -by tweeting us at ``@urbansim`` or post on the UrbanSim `forum`_. +Let us know what you are working on, or if you think you have a great use case, +by tweeting us at `@urbansim `__, posting on the UrbanSim `forum `__, or contacting us at info@urbansim.com. Installation ------------ -.. note:: - In the instructions below we will direct you to run various commands. - On Mac and Linux these should go in your standard terminal. - On Windows you may use the standard command prompt, the Anaconda - command prompt, or even Git Bash (if you have that installed). +The UrbanSim library is currently tested with Python versions 2.7, 3.5, 3.6, 3.7, and 3.8. -Anaconda -~~~~~~~~ +UrbanSim is distributed on the `Python Package Index `__ (for Pip) and on `Conda Forge `__. The official source code is hosted on `GitHub `__. (UrbanSim versions before 3.2 are on the UDST Conda channel rather than Conda Forge.) -UrbanSim is a Python library that uses a number of packages from the -scientific Python ecosystem. -The easiest way to get your own scientific Python installation is to -install `Anaconda `__, -which contains most of the libraries upon which UrbanSim depends. +You can install UrbanSim with either the Pip or Conda package manager: -UrbanSim -~~~~~~~~ +.. code-block:: python -Dependencies -^^^^^^^^^^^^ - -UrbanSim depends on the following libraries, most of which are in Anaconda: - -* `bottle `__ >= 0.12 -* `matplotlib `__ >= 1.3.1 -* `numpy `__ >= 1.8.0 -* `orca `__ >= 1.1 -* `pandas `__ >= 0.17.0 -* `patsy `__ >= 0.3.0 -* `prettytable `__ >= 0.7.2 -* `pyyaml `__ >= 3.10 -* `scipy `__ >= 0.13.3 -* `simplejson `__ >= 3.3 -* `statsmodels `__ >= 0.8.0 -* `tables `__ >= 3.1.0 -* `toolz `__ >= 0.7 -* `zbox `__ >= 1.2 - -Extras require: - -* `pandana `__ >= 0.1 - -Latest Release -^^^^^^^^^^^^^^ - -conda -##### - -`conda `__, which comes with Anaconda, is the -easiest way to install UrbanSim because it has binary installers for -all of UrbanSim's hard-to-install dependencies. -First, add the `udst channel `__ -to your conda configuration:: - - conda config --add channels udst - -Then use conda to install UrbanSim:: - - conda install urbansim - -To update to a new UrbanSim version use the ``conda update`` command:: - - conda update urbansim - -pip -### + pip install urbansim -UrbanSim can also be installed from -`PyPI `__ -via `pip `__:: +.. code-block:: python - pip install urbansim + conda install urbansim --channel conda-forge -When using this method it's best to already have most of the dependencies -installed, otherwise pip will try to download and install things like -NumPy, SciPy, and matplotlib. -If you're using Anaconda you will already have all of the hard-to-install -libraries. +Dependencies include `NumPy `__, `Pandas `__, and `Statsmodels `__, plus another UDST library: `Orca `__ for task orchestration. These will be installed automatically if needed. -To update to a new release of UrbanSim use the ``-U`` option with -``pip install``:: +When new releases of UrbanSim come out, you can upgrade like this: - pip install -U urbansim +.. code-block:: python -Development Version -^^^^^^^^^^^^^^^^^^^ + pip install urbansim --upgrade -UrbanSim can be installed from our -`development repository `__ -using `pip `__, a Python package manager. -pip is included with Anaconda so you should now be able to open a terminal -and run the following command to install UrbanSim:: +.. code-block:: python - pip install -U https://github.com/udst/urbansim/archive/master.zip + conda update urbansim --channel conda-forge -This will download urbansim and install the remaining dependencies not -included in Anaconda. +Installing from GitHub +~~~~~~~~~~~~~~~~~~~~~~ -If you need to update UrbanSim run the above command again. +You can also install UrbanSim directly from source code on GitHub, for example to use pre-release features or to modify the code yourself. The UrbanSim library is written entirely in Python; no compilation is needed. -Developer Install -^^^^^^^^^^^^^^^^^ +.. code-block:: python -If you are going to be developing on UrbanSim you will want to fork our -`GitHub repository `_ and clone -your fork to your computer. Then run ``python setup.py develop`` to install -UrbanSim in developer mode. In this mode you won't have to reinstall -UrbanSim every time you make changes. + git clone -b branch-name https://github.com/udst/urbansim.git + cd urbansim + pip install . Reporting bugs and contributing to UrbanSim ------------------------------------------- @@ -429,6 +357,3 @@ can be added to the same file so that a piece of functionality can be separated be kept together, columns together, and tables together - the organization is up to you. We hope that this flexibility inspires innovation for specific use cases, but what follows is a set of tutorials that we consider best practices. - - -.. _forum: http://discussion.urbansim.com/ \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..eba9d8a0 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,34 @@ +.. urbansim documentation master file, created by + sphinx-quickstart on Thu May 22 12:13:43 2014. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +UrbanSim +======== + +UrbanSim is a platform for building statistical models of cities and regions. These models help forecast long-range patterns in real estate development, demographics, and related outcomes, under various policy scenarios. + +This ``urbansim`` Python library is a core component. It contains tools for statistical estimation and simulation; domain-specific logic about housing markets, household relocation, and other processes; and frameworks and utilities for assembling a model. + +v3.2, released May 5, 2020 + +Contents +-------- + +.. toctree:: + :maxdepth: 2 + + gettingstarted + examples + models/index + developer/index + utils/index + maps/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/make.bat b/docs/source/make.bat similarity index 100% rename from docs/make.bat rename to docs/source/make.bat diff --git a/docs/maps/index.rst b/docs/source/maps/index.rst similarity index 100% rename from docs/maps/index.rst rename to docs/source/maps/index.rst diff --git a/docs/models/index.rst b/docs/source/models/index.rst similarity index 100% rename from docs/models/index.rst rename to docs/source/models/index.rst diff --git a/docs/models/statistical.rst b/docs/source/models/statistical.rst similarity index 100% rename from docs/models/statistical.rst rename to docs/source/models/statistical.rst diff --git a/docs/models/supplydemand.rst b/docs/source/models/supplydemand.rst similarity index 100% rename from docs/models/supplydemand.rst rename to docs/source/models/supplydemand.rst diff --git a/docs/models/transrelo.rst b/docs/source/models/transrelo.rst similarity index 100% rename from docs/models/transrelo.rst rename to docs/source/models/transrelo.rst diff --git a/docs/models/urbanchoice.rst b/docs/source/models/urbanchoice.rst similarity index 100% rename from docs/models/urbanchoice.rst rename to docs/source/models/urbanchoice.rst diff --git a/docs/models/util.rst b/docs/source/models/util.rst similarity index 100% rename from docs/models/util.rst rename to docs/source/models/util.rst diff --git a/docs/screenshots/dframe_explorer_screenshot.png b/docs/source/screenshots/dframe_explorer_screenshot.png similarity index 100% rename from docs/screenshots/dframe_explorer_screenshot.png rename to docs/source/screenshots/dframe_explorer_screenshot.png diff --git a/docs/utils/index.rst b/docs/source/utils/index.rst similarity index 100% rename from docs/utils/index.rst rename to docs/source/utils/index.rst diff --git a/docs/utils/misc.rst b/docs/source/utils/misc.rst similarity index 100% rename from docs/utils/misc.rst rename to docs/source/utils/misc.rst diff --git a/docs/utils/testing.rst b/docs/source/utils/testing.rst similarity index 100% rename from docs/utils/testing.rst rename to docs/source/utils/testing.rst diff --git a/docs/utils/yamlio.rst b/docs/source/utils/yamlio.rst similarity index 100% rename from docs/utils/yamlio.rst rename to docs/source/utils/yamlio.rst diff --git a/ez_setup.py b/ez_setup.py deleted file mode 100644 index a34fa806..00000000 --- a/ez_setup.py +++ /dev/null @@ -1,332 +0,0 @@ -#!/usr/bin/env python -"""Bootstrap setuptools installation - -To use setuptools in your package's setup.py, include this -file in the same directory and add this to the top of your setup.py:: - - from ez_setup import use_setuptools - use_setuptools() - -To require a specific version of setuptools, set a download -mirror, or use an alternate download directory, simply supply -the appropriate options to ``use_setuptools()``. - -This file can also be run as a script to install or upgrade setuptools. -""" -import os -import shutil -import sys -import tempfile -import zipfile -import optparse -import subprocess -import platform -import textwrap -import contextlib - -from distutils import log - -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen - -try: - from site import USER_SITE -except ImportError: - USER_SITE = None - -DEFAULT_VERSION = "5.7" -DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" - -def _python_cmd(*args): - """ - Return True if the command succeeded. - """ - args = (sys.executable,) + args - return subprocess.call(args) == 0 - - -def _install(archive_filename, install_args=()): - with archive_context(archive_filename): - # installing - log.warn('Installing Setuptools') - if not _python_cmd('setup.py', 'install', *install_args): - log.warn('Something went wrong during the installation.') - log.warn('See the error message above.') - # exitcode will be 2 - return 2 - - -def _build_egg(egg, archive_filename, to_dir): - with archive_context(archive_filename): - # building an egg - log.warn('Building a Setuptools egg in %s', to_dir) - _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) - # returning the result - log.warn(egg) - if not os.path.exists(egg): - raise IOError('Could not build the egg.') - - -class ContextualZipFile(zipfile.ZipFile): - """ - Supplement ZipFile class to support context manager for Python 2.6 - """ - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - def __new__(cls, *args, **kwargs): - """ - Construct a ZipFile or ContextualZipFile as appropriate - """ - if hasattr(zipfile.ZipFile, '__exit__'): - return zipfile.ZipFile(*args, **kwargs) - return super(ContextualZipFile, cls).__new__(cls) - - -@contextlib.contextmanager -def archive_context(filename): - # extracting the archive - tmpdir = tempfile.mkdtemp() - log.warn('Extracting in %s', tmpdir) - old_wd = os.getcwd() - try: - os.chdir(tmpdir) - with ContextualZipFile(filename) as archive: - archive.extractall() - - # going in the directory - subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) - os.chdir(subdir) - log.warn('Now working in %s', subdir) - yield - - finally: - os.chdir(old_wd) - shutil.rmtree(tmpdir) - - -def _do_download(version, download_base, to_dir, download_delay): - egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' - % (version, sys.version_info[0], sys.version_info[1])) - if not os.path.exists(egg): - archive = download_setuptools(version, download_base, - to_dir, download_delay) - _build_egg(egg, archive, to_dir) - sys.path.insert(0, egg) - - # Remove previously-imported pkg_resources if present (see - # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). - if 'pkg_resources' in sys.modules: - del sys.modules['pkg_resources'] - - import setuptools - setuptools.bootstrap_install_from = egg - - -def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, download_delay=15): - to_dir = os.path.abspath(to_dir) - rep_modules = 'pkg_resources', 'setuptools' - imported = set(sys.modules).intersection(rep_modules) - try: - import pkg_resources - except ImportError: - return _do_download(version, download_base, to_dir, download_delay) - try: - pkg_resources.require("setuptools>=" + version) - return - except pkg_resources.DistributionNotFound: - return _do_download(version, download_base, to_dir, download_delay) - except pkg_resources.VersionConflict as VC_err: - if imported: - msg = textwrap.dedent(""" - The required version of setuptools (>={version}) is not available, - and can't be installed while this script is running. Please - install a more recent version first, using - 'easy_install -U setuptools'. - - (Currently using {VC_err.args[0]!r}) - """).format(VC_err=VC_err, version=version) - sys.stderr.write(msg) - sys.exit(2) - - # otherwise, reload ok - del pkg_resources, sys.modules['pkg_resources'] - return _do_download(version, download_base, to_dir, download_delay) - -def _clean_check(cmd, target): - """ - Run the command to download target. If the command fails, clean up before - re-raising the error. - """ - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - if os.access(target, os.F_OK): - os.unlink(target) - raise - -def download_file_powershell(url, target): - """ - Download the file at url to target using Powershell (which will validate - trust). Raise an exception if the command cannot complete. - """ - target = os.path.abspath(target) - ps_cmd = ( - "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " - "[System.Net.CredentialCache]::DefaultCredentials; " - "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" - % vars() - ) - cmd = [ - 'powershell', - '-Command', - ps_cmd, - ] - _clean_check(cmd, target) - -def has_powershell(): - if platform.system() != 'Windows': - return False - cmd = ['powershell', '-Command', 'echo test'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True - -download_file_powershell.viable = has_powershell - -def download_file_curl(url, target): - cmd = ['curl', url, '--silent', '--output', target] - _clean_check(cmd, target) - -def has_curl(): - cmd = ['curl', '--version'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True - -download_file_curl.viable = has_curl - -def download_file_wget(url, target): - cmd = ['wget', url, '--quiet', '--output-document', target] - _clean_check(cmd, target) - -def has_wget(): - cmd = ['wget', '--version'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True - -download_file_wget.viable = has_wget - -def download_file_insecure(url, target): - """ - Use Python to download the file, even though it cannot authenticate the - connection. - """ - src = urlopen(url) - try: - # Read all the data in one block. - data = src.read() - finally: - src.close() - - # Write all the data in one block to avoid creating a partial file. - with open(target, "wb") as dst: - dst.write(data) - -download_file_insecure.viable = lambda: True - -def get_best_downloader(): - downloaders = ( - download_file_powershell, - download_file_curl, - download_file_wget, - download_file_insecure, - ) - viable_downloaders = (dl for dl in downloaders if dl.viable()) - return next(viable_downloaders, None) - -def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, delay=15, downloader_factory=get_best_downloader): - """ - Download setuptools from a specified location and return its filename - - `version` should be a valid setuptools version number that is available - as an sdist for download under the `download_base` URL (which should end - with a '/'). `to_dir` is the directory where the egg will be downloaded. - `delay` is the number of seconds to pause before an actual download - attempt. - - ``downloader_factory`` should be a function taking no arguments and - returning a function for downloading a URL to a target. - """ - # making sure we use the absolute path - to_dir = os.path.abspath(to_dir) - zip_name = "setuptools-%s.zip" % version - url = download_base + zip_name - saveto = os.path.join(to_dir, zip_name) - if not os.path.exists(saveto): # Avoid repeated downloads - log.warn("Downloading %s", url) - downloader = downloader_factory() - downloader(url, saveto) - return os.path.realpath(saveto) - -def _build_install_args(options): - """ - Build the arguments to 'python setup.py install' on the setuptools package - """ - return ['--user'] if options.user_install else [] - -def _parse_args(): - """ - Parse the command line for options - """ - parser = optparse.OptionParser() - parser.add_option( - '--user', dest='user_install', action='store_true', default=False, - help='install in user site package (requires Python 2.6 or later)') - parser.add_option( - '--download-base', dest='download_base', metavar="URL", - default=DEFAULT_URL, - help='alternative URL from where to download the setuptools package') - parser.add_option( - '--insecure', dest='downloader_factory', action='store_const', - const=lambda: download_file_insecure, default=get_best_downloader, - help='Use internal, non-validating downloader' - ) - parser.add_option( - '--version', help="Specify which version to download", - default=DEFAULT_VERSION, - ) - options, args = parser.parse_args() - # positional arguments are ignored - return options - -def main(): - """Install or upgrade setuptools and EasyInstall""" - options = _parse_args() - archive = download_setuptools( - version=options.version, - download_base=options.download_base, - downloader_factory=options.downloader_factory, - ) - return _install(archive, _build_install_args(options)) - -if __name__ == '__main__': - sys.exit(main()) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..635cd932 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,18 @@ +# Additional requirements for development and testing + +bottle +matplotlib +pandana +simplejson + +# testing +coveralls +jupyter +pycodestyle +pytest < 4.0 +pytest-cov + +# building documentation +numpydoc +sphinx +sphinx-rtd-theme diff --git a/scripts/cache_to_hdf5.py b/scripts/cache_to_hdf5.py deleted file mode 100755 index d9cdece9..00000000 --- a/scripts/cache_to_hdf5.py +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function - -import argparse -import glob -import os -import sys - -import numpy as np -import pandas as pd - - -def cache_to_df(dir_path): - """ - Convert a directory of binary array data files to a Pandas DataFrame. - - Parameters - ---------- - dir_path : str - - - """ - table = {} - for attrib in glob.glob(os.path.join(dir_path, '*')): - attrib_name, attrib_ext = os.path.splitext(os.path.basename(attrib)) - if attrib_ext == '.lf8': - attrib_data = np.fromfile(attrib, np.float64) - table[attrib_name] = attrib_data - - elif attrib_ext == '.lf4': - attrib_data = np.fromfile(attrib, np.float32) - table[attrib_name] = attrib_data - - elif attrib_ext == '.li2': - attrib_data = np.fromfile(attrib, np.int16) - table[attrib_name] = attrib_data - - elif attrib_ext == '.li4': - attrib_data = np.fromfile(attrib, np.int32) - table[attrib_name] = attrib_data - - elif attrib_ext == '.li8': - attrib_data = np.fromfile(attrib, np.int64) - table[attrib_name] = attrib_data - - elif attrib_ext == '.ib1': - attrib_data = np.fromfile(attrib, np.bool_) - table[attrib_name] = attrib_data - - elif attrib_ext.startswith('.iS'): - length_string = int(attrib_ext[3:]) - attrib_data = np.fromfile(attrib, ('a' + str(length_string))) - table[attrib_name] = attrib_data - - else: - print('Array {} is not a recognized data type'.format(attrib)) - - df = pd.DataFrame(table) - return df - - -DIRECTORIES = { - 'parcels', 'buildings', 'households', 'jobs', 'zones', 'travel_data', - 'annual_employment_control_totals', 'annual_household_control_totals', - 'annual_household_relocation_rates', 'annual_job_relocation_rates', - 'building_sqft_per_job', 'building_types', 'counties', 'target_vacancies', - 'development_event_history' -} - - -def convert_dirs(base_dir, hdf_name, complib=None, complevel=0): - """ - Convert nested set of directories to - - """ - print('Converting directories in {}'.format(base_dir)) - - dirs = glob.glob(os.path.join(base_dir, '*')) - dirs = {d for d in dirs if os.path.basename(d) in DIRECTORIES} - if not dirs: - raise RuntimeError('No direcotries found matching known data.') - - store = pd.HDFStore( - hdf_name, mode='w', complevel=complevel, complib=complib) - - for dirpath in dirs: - dirname = os.path.basename(dirpath) - - print(dirname) - df = cache_to_df(dirpath) - - if dirname == 'travel_data': - keys = ['from_zone_id', 'to_zone_id'] - elif dirname == 'annual_employment_control_totals': - keys = ['sector_id', 'year', 'home_based_status'] - elif dirname == 'annual_job_relocation_rates': - keys = ['sector_id'] - elif dirname == 'annual_household_control_totals': - keys = ['year'] - elif dirname == 'annual_household_relocation_rates': - keys = ['age_of_head_max', 'age_of_head_min', - 'income_min', 'income_max'] - elif dirname == 'building_sqft_per_job': - keys = ['zone_id', 'building_type_id'] - elif dirname == 'counties': - keys = ['county_id'] - elif dirname == 'development_event_history': - keys = ['building_id'] - elif dirname == 'target_vacancies': - keys = ['building_type_id', 'year'] - else: - keys = [dirname[:-1] + '_id'] - - if dirname != 'annual_household_relocation_rates': - df = df.set_index(keys) - - for colname in df.columns: - if df[colname].dtype == np.float64: - df[colname] = df[colname].astype(np.float32) - elif df[colname].dtype == np.int64: - df[colname] = df[colname].astype(np.int32) - else: - df[colname] = df[colname] - - df.info() - print(os.linesep) - store.put(dirname, df) - - store.close() - - -def parse_args(args=None): - parser = argparse.ArgumentParser( - description=( - 'Convert nested set of directories containing binary ' - 'array data to an HDF5 file made from Pandas DataFrames.')) - parser.add_argument('base_dir', help='Base data directory for conversion.') - parser.add_argument('hdf_name', help='Name of output HDF5 file.') - parser.add_argument('-c', '--complib', - help=('Compression library to use, if any. ' - "Can be one of " - "'zlib', 'bzip2', 'lzo', 'blosc'.")) - parser.add_argument('-l', '--complevel', type=int, default=0, - help='Compression level to use.') - return parser.parse_args(args) - - -def main(args=None): - args = parse_args(args) - convert_dirs(args.base_dir, args.hdf_name, args.complib, args.complevel) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/scripts/view_hdf5.py b/scripts/view_hdf5.py deleted file mode 100644 index 47e9f0f4..00000000 --- a/scripts/view_hdf5.py +++ /dev/null @@ -1,18 +0,0 @@ -import json -import sys - -import pandas as pd - -args = sys.argv[1:] - -assert len(args) == 1 - -store = pd.HDFStore(args[0], "r") - -print store - -for key in store.keys(): - print "\n\nTABLENAME", key - print store[key] - print store[key].dtypes - print store[key].describe() diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6aeda162..00000000 --- a/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[pep8] -exclude = texttable.py -max-line-length = 100 diff --git a/setup.py b/setup.py index c5c50bd9..fa4b5c74 100644 --- a/setup.py +++ b/setup.py @@ -1,31 +1,28 @@ -# Install setuptools if not installed. -try: - import setuptools -except ImportError: - from ez_setup import use_setuptools - use_setuptools() - from setuptools import setup, find_packages - # read README as the long description with open('README.rst', 'r') as f: long_description = f.read() setup( name='urbansim', - version='3.1.1', - description='Tool for modeling metropolitan real estate markets', + version='3.2', + description='Platform for building statistical models of cities and regions', long_description=long_description, author='UrbanSim Inc.', author_email='info@urbansim.com', license='BSD', url='https://github.com/udst/urbansim', classifiers=[ - 'Development Status :: 4 - Beta', + 'Intended Audience :: Science/Research', + 'Topic :: Scientific/Engineering :: Information Analysis', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: BSD License' ], package_data={ @@ -33,22 +30,15 @@ }, packages=find_packages(exclude=['*.tests']), install_requires=[ - 'bottle>=0.12', - 'matplotlib>=1.3.1', - 'numpy>=1.8.0', - 'orca>=1.1', - 'pandas>=0.17.0', - 'patsy>=0.3.0', - 'prettytable>=0.7.2', - 'pyyaml>=3.10', - 'scipy>=0.13.3', - 'simplejson>=3.3', - 'statsmodels>=0.8.0', - 'tables>=3.1.0', - 'toolz>=0.7.0', - 'zbox>=1.2' - ], - extras_require={ - 'pandana': ['pandana==0.3.0'] - } + 'numpy >= 1.8.0', + 'orca >= 1.1', + 'pandas >= 0.17.0', + 'patsy >= 0.4.1', + 'prettytable >= 0.7.2', + 'pyyaml >= 3.10', + 'scipy >= 1.0', + 'statsmodels >= 0.8, <0.11; python_version <"3.6"', + 'statsmodels >= 0.8; python_version >="3.6"', + 'toolz >= 0.8.1' + ] ) diff --git a/urbansim/__init__.py b/urbansim/__init__.py index abd22615..59cbdc33 100644 --- a/urbansim/__init__.py +++ b/urbansim/__init__.py @@ -1 +1 @@ -__version__ = version = '3.1.1' +__version__ = version = '3.2' diff --git a/urbansim/accounts.py b/urbansim/accounts.py index a9c08061..e3ed7df6 100644 --- a/urbansim/accounts.py +++ b/urbansim/accounts.py @@ -5,7 +5,7 @@ from collections import namedtuple import pandas as pd -from zbox import toolz as tz +import toolz as tz Transaction = namedtuple('Transaction', ('amount', 'subaccount', 'metadata')) diff --git a/urbansim/models/dcm.py b/urbansim/models/dcm.py index b649ad1d..f55c28d8 100644 --- a/urbansim/models/dcm.py +++ b/urbansim/models/dcm.py @@ -12,7 +12,7 @@ import pandas as pd from patsy import dmatrix from prettytable import PrettyTable -from zbox import toolz as tz +import toolz as tz from . import util from ..exceptions import ModelEvaluationError @@ -413,14 +413,14 @@ def fit(self, choosers, alternatives, current_choice): model_design = dmatrix( self.str_model_expression, data=merged, return_type='dataframe') - if len(merged) != model_design.as_matrix().shape[0]: + if len(merged) != model_design.values.shape[0]: raise ModelEvaluationError( 'Estimated data does not have the same length as input. ' 'This suggests there are null values in one or more of ' 'the input columns.') self.log_likelihoods, self.fit_parameters = mnl.mnl_estimate( - model_design.as_matrix(), chosen, self.sample_size) + model_design.values, chosen, self.sample_size) self.fit_parameters.index = model_design.columns logger.debug('finish: fit LCM model {}'.format(self.name)) @@ -523,7 +523,7 @@ def probabilities(self, choosers, alternatives, filter_tables=True): model_design = dmatrix( self.str_model_expression, data=merged, return_type='dataframe') - if len(merged) != model_design.as_matrix().shape[0]: + if len(merged) != model_design.values.shape[0]: raise ModelEvaluationError( 'Simulated data does not have the same length as input. ' 'This suggests there are null values in one or more of ' @@ -542,7 +542,7 @@ def probabilities(self, choosers, alternatives, filter_tables=True): numalts = sample_size probabilities = mnl.mnl_simulate( - model_design.as_matrix(), + model_design.values, coeffs, numalts=numalts, returnprobs=True) diff --git a/urbansim/models/regression.py b/urbansim/models/regression.py index 54298226..b80d363b 100644 --- a/urbansim/models/regression.py +++ b/urbansim/models/regression.py @@ -12,7 +12,7 @@ import statsmodels.formula.api as smf from patsy import dmatrix from prettytable import PrettyTable -from zbox import toolz as tz +import toolz as tz from . import util from ..exceptions import ModelEvaluationError diff --git a/urbansim/models/tests/test_dcm.py b/urbansim/models/tests/test_dcm.py index fa60ba06..e1254dba 100644 --- a/urbansim/models/tests/test_dcm.py +++ b/urbansim/models/tests/test_dcm.py @@ -222,10 +222,10 @@ def test_mnl_dcm_yaml(basic_dcm, choosers, alternatives): 'fit_parameters': None } - assert yaml.load(basic_dcm.to_yaml()) == expected_dict + assert yaml.safe_load(basic_dcm.to_yaml()) == expected_dict new_mod = dcm.MNLDiscreteChoiceModel.from_yaml(basic_dcm.to_yaml()) - assert yaml.load(new_mod.to_yaml()) == expected_dict + assert yaml.safe_load(new_mod.to_yaml()) == expected_dict basic_dcm.fit(choosers, alternatives, 'thing_id') @@ -233,7 +233,7 @@ def test_mnl_dcm_yaml(basic_dcm, choosers, alternatives): del expected_dict['log_likelihoods'] del expected_dict['fit_parameters'] - actual_dict = yaml.load(basic_dcm.to_yaml()) + actual_dict = yaml.safe_load(basic_dcm.to_yaml()) assert isinstance(actual_dict.pop('log_likelihoods'), dict) assert isinstance(actual_dict.pop('fit_parameters'), dict) @@ -524,10 +524,10 @@ def test_mnl_dcm_segmented_yaml(grouped_choosers, alternatives): } } - assert yaml.load(group.to_yaml()) == expected_dict + assert yaml.safe_load(group.to_yaml()) == expected_dict new_seg = dcm.SegmentedMNLDiscreteChoiceModel.from_yaml(group.to_yaml()) - assert yaml.load(new_seg.to_yaml()) == expected_dict + assert yaml.safe_load(new_seg.to_yaml()) == expected_dict group.fit(grouped_choosers, alternatives, 'thing_id') @@ -539,7 +539,7 @@ def test_mnl_dcm_segmented_yaml(grouped_choosers, alternatives): del expected_dict['models']['y']['fit_parameters'] del expected_dict['models']['y']['log_likelihoods'] - actual_dict = yaml.load(group.to_yaml()) + actual_dict = yaml.safe_load(group.to_yaml()) assert isinstance(actual_dict['models']['x'].pop('fit_parameters'), dict) assert isinstance(actual_dict['models']['x'].pop('log_likelihoods'), dict) assert isinstance(actual_dict['models']['y'].pop('fit_parameters'), dict) diff --git a/urbansim/models/tests/test_regression.py b/urbansim/models/tests/test_regression.py index bc55ad58..de1743f2 100644 --- a/urbansim/models/tests/test_regression.py +++ b/urbansim/models/tests/test_regression.py @@ -211,7 +211,7 @@ def setup_method(self, method): def test_string(self): test_yaml = self.model.to_yaml() - assert_dict_specs_equal(yaml.load(test_yaml), self.expected_dict) + assert_dict_specs_equal(yaml.safe_load(test_yaml), self.expected_dict) model = regression.RegressionModel.from_yaml(yaml_str=test_yaml) assert isinstance(model, regression.RegressionModel) @@ -220,7 +220,7 @@ def test_buffer(self): test_buffer = StringIO() self.model.to_yaml(str_or_buffer=test_buffer) assert_dict_specs_equal( - yaml.load(test_buffer.getvalue()), self.expected_dict) + yaml.safe_load(test_buffer.getvalue()), self.expected_dict) test_buffer.seek(0) model = regression.RegressionModel.from_yaml(str_or_buffer=test_buffer) @@ -233,7 +233,7 @@ def test_file(self): self.model.to_yaml(str_or_buffer=test_file) with open(test_file) as f: - assert_dict_specs_equal(yaml.load(f), self.expected_dict) + assert_dict_specs_equal(yaml.safe_load(f), self.expected_dict) model = regression.RegressionModel.from_yaml(str_or_buffer=test_file) assert isinstance(model, regression.RegressionModel) @@ -375,10 +375,10 @@ def test_SegmentedRegressionModel_yaml(groupby_df): } } - assert yaml.load(seg.to_yaml()) == expected_dict + assert yaml.safe_load(seg.to_yaml()) == expected_dict new_seg = regression.SegmentedRegressionModel.from_yaml(seg.to_yaml()) - assert yaml.load(new_seg.to_yaml()) == expected_dict + assert yaml.safe_load(new_seg.to_yaml()) == expected_dict seg.fit(groupby_df) @@ -392,7 +392,7 @@ def test_SegmentedRegressionModel_yaml(groupby_df): del expected_dict['models']['y']['fit_rsquared'] del expected_dict['models']['y']['fit_rsquared_adj'] - actual_dict = yaml.load(seg.to_yaml()) + actual_dict = yaml.safe_load(seg.to_yaml()) assert isinstance(actual_dict['models']['x'].pop('fit_parameters'), dict) assert isinstance(actual_dict['models']['x'].pop('fit_rsquared'), float) assert isinstance( diff --git a/urbansim/models/util.py b/urbansim/models/util.py index 799f1d94..868ea58b 100644 --- a/urbansim/models/util.py +++ b/urbansim/models/util.py @@ -14,7 +14,7 @@ import numpy as np import pandas as pd import patsy -from zbox import toolz as tz +import toolz as tz from ..utils.logutil import log_start_finish diff --git a/urbansim/urbanchoice/tests/test_interaction.py b/urbansim/urbanchoice/tests/test_interaction.py index 938edcf4..aeca68a5 100644 --- a/urbansim/urbanchoice/tests/test_interaction.py +++ b/urbansim/urbanchoice/tests/test_interaction.py @@ -35,8 +35,8 @@ def test_interaction_dataset_sim(choosers, alternatives): assert len(merged) == len(choosers) * len(alternatives) npt.assert_array_equal(merged.index.values, sample) - assert list(merged.columns) == [ - 'var2', 'var3', 'join_index', 'thing_id', 'var1'] + assert set(list(merged.columns)) == set([ + 'var2', 'var3', 'join_index', 'thing_id', 'var1']) npt.assert_array_equal( merged['var1'].values, choosers['var1'].values.repeat(len(alternatives))) diff --git a/urbansim/urbanchoice/tests/test_mnl.py b/urbansim/urbanchoice/tests/test_mnl.py index a939e462..ff6ef4cf 100644 --- a/urbansim/urbanchoice/tests/test_mnl.py +++ b/urbansim/urbanchoice/tests/test_mnl.py @@ -110,12 +110,12 @@ def choosers_dm(choosers, test_data): @pytest.fixture def fit_coeffs(dm, chosen, num_alts): - log_like, fit = mnl.mnl_estimate(dm.as_matrix(), chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values, chosen, num_alts) return fit.Coefficient.values def test_mnl_estimate(dm, chosen, num_alts, test_data): - log_like, fit = mnl.mnl_estimate(dm.as_matrix(), chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values, chosen, num_alts) result = pd.Series(fit.Coefficient.values, index=dm.columns) result, expected = result.align(test_data['est_expected']) npt.assert_allclose(result.values, expected.values, rtol=1e-4) @@ -134,10 +134,10 @@ def test_mnl_simulate(dm, fit_coeffs, num_alts, test_data, choosers_dm): # now test with real data probs = mnl.mnl_simulate( - choosers_dm.as_matrix(), fit_coeffs, num_alts, returnprobs=True) + choosers_dm.values, fit_coeffs, num_alts, returnprobs=True) results = pd.DataFrame(probs, columns=test_data['sim_expected'].columns) results, expected = results.align(test_data['sim_expected']) - npt.assert_allclose(results.as_matrix(), expected.as_matrix(), rtol=1e-4) + npt.assert_allclose(results.values, expected.values, rtol=1e-4) def test_alternative_specific_coeffs(num_alts): @@ -193,7 +193,7 @@ def test_alternative_specific_coeffs(num_alts): 'boat:(intercept)', 'charter:(intercept)', 'pier:(intercept)', 'boat:income', 'charter:income', 'pier:income']) - log_like, fit = mnl.mnl_estimate(dm.as_matrix(), fish_chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values, fish_chosen, num_alts) result = pd.Series(fit.Coefficient.values, index=dm.columns) result, expected = result.align(expected) npt.assert_allclose(result.values, expected.values, rtol=1e-4) @@ -206,7 +206,7 @@ def test_alternative_specific_coeffs(num_alts): fit_coeffs = fit.Coefficient.values probs = mnl.mnl_simulate( - choosers_dm.as_matrix(), fit_coeffs, num_alts, returnprobs=True) + choosers_dm.values, fit_coeffs, num_alts, returnprobs=True) results = pd.DataFrame(probs, columns=expected.columns) results, expected = results.align(expected) - npt.assert_allclose(results.as_matrix(), expected.as_matrix(), rtol=1e-4) + npt.assert_allclose(results.values, expected.values, rtol=1e-4) diff --git a/urbansim/utils/misc.py b/urbansim/utils/misc.py index b2aadfd3..f5d410a6 100644 --- a/urbansim/utils/misc.py +++ b/urbansim/utils/misc.py @@ -8,7 +8,7 @@ import numpy as np import pandas as pd -from zbox import toolz as tz +import toolz as tz def _mkifnotexists(folder): @@ -138,7 +138,7 @@ def compute_range(travel_data, attr, travel_time_attr, dist, agg=np.sum): """ travel_data = travel_data.reset_index(level=1) travel_data = travel_data[travel_data[travel_time_attr] < dist] - travel_data["attr"] = attr[travel_data.to_zone_id].values + travel_data["attr"] = attr.reindex(travel_data.to_zone_id, fill_value=0).values return travel_data.groupby(level=0).attr.apply(agg) @@ -357,7 +357,7 @@ def series64bitto32bit(s): def _pandassummarytojson(v, ndigits=3): - return {i: round(float(v.ix[i]), ndigits) for i in v.index} + return {i: round(float(v.loc[i]), ndigits) for i in v.index} def pandasdfsummarytojson(df, ndigits=3): diff --git a/urbansim/utils/networks.py b/urbansim/utils/networks.py index 00549ffb..5e4757e8 100644 --- a/urbansim/utils/networks.py +++ b/urbansim/utils/networks.py @@ -15,7 +15,7 @@ def from_yaml(net, cfgname): print("Computing accessibility variables") - cfg = yaml.load(open(misc.config(cfgname))) + cfg = yaml.safe_load(open(misc.config(cfgname))) nodes = pd.DataFrame(index=net.node_ids) diff --git a/urbansim/utils/tests/test_testing.py b/urbansim/utils/tests/test_testing.py index 642e8a56..f2ad800f 100644 --- a/urbansim/utils/tests/test_testing.py +++ b/urbansim/utils/tests/test_testing.py @@ -17,36 +17,36 @@ def test_frames_equal_mismatched_columns(): expected = pd.DataFrame({'a': [1]}) actual = pd.DataFrame({'b': [2]}) - with pytest.raises(AssertionError) as info: + try: testing.assert_frames_equal(actual, expected) - - assert str(info.value) == "Expected column 'a' not found." + except AssertionError: + pass + else: + raise AssertionError def test_frames_equal_mismatched_rows(): expected = pd.DataFrame({'a': [1]}, index=[0]) actual = pd.DataFrame({'a': [1]}, index=[1]) - with pytest.raises(AssertionError) as info: + try: testing.assert_frames_equal(actual, expected) - - assert str(info.value) == "Expected row 0 not found." + except AssertionError: + pass + else: + raise AssertionError def test_frames_equal_mismatched_items(): expected = pd.DataFrame({'a': [1]}) actual = pd.DataFrame({'a': [2]}) - with pytest.raises(AssertionError) as info: + try: testing.assert_frames_equal(actual, expected) - - assert str(info.value) == """ -Items are not equal: - ACTUAL: 2 - DESIRED: 1 - -Column: 'a' -Row: 0""" + except AssertionError: + pass + else: + raise AssertionError def test_frames_equal(): diff --git a/urbansim/utils/tests/test_yamlio.py b/urbansim/utils/tests/test_yamlio.py index d5804bed..f17cdecc 100644 --- a/urbansim/utils/tests/test_yamlio.py +++ b/urbansim/utils/tests/test_yamlio.py @@ -123,7 +123,7 @@ def test_series_to_yaml_safe_int_index(): assert d == {0: 100, 1: 101, 2: 102} y = yaml.dump(d, default_flow_style=False) - assert_series_equal(pd.Series(yaml.load(y)), s) + assert_series_equal(pd.Series(yaml.safe_load(y)), s) def test_series_to_yaml_safe_str_index(): @@ -133,7 +133,7 @@ def test_series_to_yaml_safe_str_index(): assert d == {'x': 'a', 'y': 'b', 'z': 'c'} y = yaml.dump(d, default_flow_style=False) - assert_series_equal(pd.Series(yaml.load(y)), s) + assert_series_equal(pd.Series(yaml.safe_load(y)), s) def test_frame_to_yaml_safe(): @@ -146,7 +146,7 @@ def test_frame_to_yaml_safe(): assert d == {'col1': {0: 100, 1: 200, 2: 300}, 'col2': {0: 'a', 1: 'b', 2: 'c'}} y = yaml.dump(d, default_flow_style=False) - assert_dfs_equal(pd.DataFrame(yaml.load(y)), df) + assert_dfs_equal(pd.DataFrame(yaml.safe_load(y)), df) def test_ordered_dict(): diff --git a/urbansim/utils/yamlio.py b/urbansim/utils/yamlio.py index 2e8391a6..2a7ff78a 100644 --- a/urbansim/utils/yamlio.py +++ b/urbansim/utils/yamlio.py @@ -84,7 +84,7 @@ def to_scalar_safe(obj): Convert a numpy data type to a standard python scalar. """ try: - return np.asscalar(obj) + return obj.item() except Exception: return obj @@ -220,7 +220,7 @@ def yaml_to_dict(yaml_str=None, str_or_buffer=None, ordered=False): if ordered: loader = __ordered_load else: - loader = yaml.load + loader = yaml.safe_load if yaml_str: d = loader(yaml_str)