Merge pull request #91 from yanlesin/v1

V1
yanlesin · May 4, 2024 · 5f51fc2 · 5f51fc2
2 parents 0e53b3d + b0e7cb0
commit 5f51fc2
Show file tree

Hide file tree

Showing 13 changed files with 52 additions and 16 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SEC13Flist
-Title: Routines to Work with Official List of SEC Section 13(f) Securities and Security Identifiers
-Version: 0.3.6
+Title: Functions to Work with Official List of SEC Section 13(f) Securities and Security Identifiers
+Version: 1.0.0
 Authors@R: 
  person(given = "Yan",
  family = "Lyesin",
@@ -14,11 +14,13 @@ Imports:
  Rcpp
 License: CC0
 Encoding: UTF-8
-LazyData: true
-RoxygenNote: 7.2.3
+Roxygen: list(markdown = TRUE)
+RoxygenNote: 7.3.1
 URL: https://github.com/yanlesin/SEC13Flist
 BugReports: https://github.com/yanlesin/SEC13Flist/issues
 LinkingTo: 
  Rcpp
 Suggests: 
  testthat (>= 2.1.0)
+Depends: 
+ R (>= 4.1)
diff --git a/NEWS.md b/NEWS.md
@@ -1,5 +1,11 @@
 # SEC13Flist 
 
+## 1.0.0
+
+* significant change: `SEC_13F_list` function requires user agent setup for reading data from sec.gov website. See [https://www.sec.gov/os/accessing-edgar-data](https://www.sec.gov/os/accessing-edgar-data)
+* fixed issue with 2023 Q4 list parsing
+* modified tests to include user agent information for sec.gov access
+
 ## 0.3.6
 
 * fixed issue with isCusip function incorrectly returned FALSE for certain CUSIPs (issue #75)

diff --git a/R/sec_13f_list.R b/R/sec_13f_list.R
@@ -17,7 +17,7 @@
 #' }
 #' @export
 #' @examples
-#' \donttest{library(SEC13Flist)
+#' \dontrun{library(SEC13Flist)
 #' SEC_13F_list_2018_Q3 <- SEC_13F_list(2018,3) #Parse list for Q3 2018 without progress indicator
 #' SEC_13F_list_2018_Q3_ <- SEC_13F_list(2018,3,TRUE) #Parse list with progress indicator
 #' SEC_13F_list_current <- SEC_13F_list() #Parse current list from SEC.gov

diff --git a/R/utils.R b/R/utils.R
@@ -232,6 +232,17 @@ process_file_func <- function(text, YEAR_, QUARTER_) {
  text2$STATUS = ifelse(text2$STATUS == "RG ADDED",
  substr(text2$STATUS, 6, 10),
  text2$STATUS)
+#2023 Q4 report 025072158
+ text2$ISSUER_DESCRIPTION = ifelse(
+ text2$STATUS == "RG",
+ trimws(paste0(
+ text2$ISSUER_DESCRIPTION, substr(text2$STATUS, 1, 2)
+ ), "both"),
+ text2$ISSUER_DESCRIPTION
+ )
+ text2$STATUS = ifelse(text2$STATUS == "RG",
+ "",
+ text2$STATUS)
 
  text2 <- subset(text2, !grepl("CUSIP", text2$CUSIP))
 

diff --git a/README.Rmd b/README.Rmd
@@ -15,15 +15,20 @@ knitr::opts_chunk$set(
 <!-- badges: start -->
 [![R build status](https://github.com/yanlesin/SEC13Flist/workflows/R-CMD-check/badge.svg)](https://github.com/yanlesin/SEC13Flist/actions)
 [![codecov](https://codecov.io/github/yanlesin/SEC13Flist/branch/master/graphs/badge.svg)](https://codecov.io/gh/yanlesin/SEC13Flist/branch/master)
+[![SEC13Flist status badge](https://yanlesin.r-universe.dev/badges/SEC13Flist)](https://yanlesin.r-universe.dev/SEC13Flist)
 <!-- badges: end -->
 
 # SEC13Flist
 
-The goal of SEC13Flist package is to provide routines to work with official list of Section 13(f) Securities.
+The goal of SEC13Flist package is to provide functions to work with official list of Section 13(f) Securities.
 
-Functions `SEC_13F_list` and `SEC_13F_list_local` parses PDF list from [SEC.gov](https://www.sec.gov/divisions/investment/13flists.htm) based on supplied year and quarter and returns data frame with list of securities, maintaining the same structure as official list. Functions appends YEAR and QUARTER columns to the list. Returned dataframe could be customized and filtered according to your needs. 
+Functions `SEC_13F_list` and `SEC_13F_list_local` parses PDF list from [SEC.gov](https://www.sec.gov/divisions/investment/13flists.htm) based on supplied year and quarter and returns data frame with list of securities, maintaining the same structure as official list. Functions appends YEAR and QUARTER columns to the list. Returned data frame could be customized and filtered according to your needs. 
 
-`SEC_13F_list` function reaches to [SEC.gov](https://www.sec.gov/divisions/investment/13flists.htm) website and requires tweaks if landing page changes. In case of a breaking change on landing page, you can use `SEC_13F_list_local` function to parse file downloaded to local folder manually.
+`SEC_13F_list` function reaches to [SEC.gov](https://www.sec.gov/divisions/investment/13flists.htm) website and requires tweaks if landing page changes. In case of a breaking change on landing page, you can use `SEC_13F_list_local` function to parse file downloaded to local folder. 
+
+`SEC_13F_list` function requires setup of user agent prior to attempting download from sec.gov website. For details how to setup user agent and maximum request rate please refer to [https://www.sec.gov/os/accessing-edgar-data](https://www.sec.gov/os/accessing-edgar-data).
+
+User agent could be setup via `options(HTTPUserAgent=...)`.
 
 Functions `isCusip`, `isSedol`, and `isIsin` verify checksum digit of security identifiers based on leading characters of the identifier (except last checksum digit). Functions returns `TRUE`/`FALSE` for correct/incorrect identifier. 
 
@@ -41,7 +46,7 @@ remotes::install_github("yanlesin/SEC13Flist")
 
 
 `CUSIP`: chr - CUSIP number of the security 
-`HAS_LISTED_OPTION`: chr - An asterisk idicates that security having a listed option and each option is individually listed with its own CUSIP number immediately below the name of the security having the option 
+`HAS_LISTED_OPTION`: chr - An asterisk indicates that security having a listed option and each option is individually listed with its own CUSIP number immediately below the name of the security having the option 
 `ISSUER_NAME`: chr - Issuer Name 
 `ISSUER_DESCRIPTION`: chr - Issuer Description 
 `STATUS`: chr - "ADDED" (The security has become a Section 13(f) security) or "DELETED" (The security ceases to be a 13(f) security since the date of the last list) 

diff --git a/README.md b/README.md
@@ -5,25 +5,32 @@
 [![R build
 status](https://github.com/yanlesin/SEC13Flist/workflows/R-CMD-check/badge.svg)](https://github.com/yanlesin/SEC13Flist/actions)
 [![codecov](https://codecov.io/github/yanlesin/SEC13Flist/branch/master/graphs/badge.svg)](https://codecov.io/gh/yanlesin/SEC13Flist/branch/master)
+[![SEC13Flist status
+badge](https://yanlesin.r-universe.dev/badges/SEC13Flist)](https://yanlesin.r-universe.dev/SEC13Flist)
 <!-- badges: end -->
 
 # SEC13Flist
 
-The goal of SEC13Flist package is to provide routines to work with
+The goal of SEC13Flist package is to provide functions to work with
 official list of Section 13(f) Securities.
 
 Functions `SEC_13F_list` and `SEC_13F_list_local` parses PDF list from
 [SEC.gov](https://www.sec.gov/divisions/investment/13flists.htm) based
 on supplied year and quarter and returns data frame with list of
 securities, maintaining the same structure as official list. Functions
-appends YEAR and QUARTER columns to the list. Returned dataframe could
+appends YEAR and QUARTER columns to the list. Returned data frame could
 be customized and filtered according to your needs.
 
 `SEC_13F_list` function reaches to
 [SEC.gov](https://www.sec.gov/divisions/investment/13flists.htm) website
 and requires tweaks if landing page changes. In case of a breaking
 change on landing page, you can use `SEC_13F_list_local` function to
-parse file downloaded to local folder manually.
+parse file downloaded to local folder. Function also requires setup of
+user agent prior to attempting download from sec.gov website. For
+details how to setup user agent and maximum request rate please refer to
+<https://www.sec.gov/os/accessing-edgar-data>.
+
+User agent could be setup via `options(HTTPUserAgent=...)`.
 
 Functions `isCusip`, `isSedol`, and `isIsin` verify checksum digit of
 security identifiers based on leading characters of the identifier
@@ -51,7 +58,7 @@ remotes::install_github("yanlesin/SEC13Flist")
 ## Description of returned data for `SEC_13F_list`
 
 `CUSIP`: chr - CUSIP number of the security 
-`HAS_LISTED_OPTION`: chr - An asterisk idicates that security having a
+`HAS_LISTED_OPTION`: chr - An asterisk indicates that security having a
 listed option and each option is individually listed with its own CUSIP
 number immediately below the name of the security having the option 
 `ISSUER_NAME`: chr - Issuer Name 

diff --git a/man/SEC_13F_list.Rd b/man/SEC_13F_list.Rd
diff --git a/man/SEC_13F_list_local.Rd b/man/SEC_13F_list_local.Rd
diff --git a/tests/testthat/test-SEC_row_count.R b/tests/testthat/test-SEC_row_count.R
@@ -1,4 +1,5 @@
 test_that("Parsed row count equal to total row count per PDF list", {
+ options(HTTPUserAgent="Yan Lyesin, SEC13Flist Package for R, [email protected] ")
  url_SEC <- "https://www.sec.gov/divisions/investment/13flists.htm"
 
  current_list_url <- rvest::html_attr(rvest::html_elements(

diff --git a/tests/testthat/test-SEC_row_count_2004.R b/tests/testthat/test-SEC_row_count_2004.R
@@ -1,4 +1,5 @@
 test_that("Parsed row count equal to total row count per PDF list for 2004 Q1", {
+ options(HTTPUserAgent="Yan Lyesin, SEC13Flist Package for R, [email protected] ")
  url_SEC <- "https://www.sec.gov/divisions/investment/13flists.htm"
 
  current_list_url <- rvest::html_attr(rvest::html_elements(

diff --git a/tests/testthat/test-has_listed_options.R b/tests/testthat/test-has_listed_options.R
@@ -1,4 +1,5 @@
 test_that("list of possible values for HAS_LISTED_OPTION column", {
+ options(HTTPUserAgent="Yan Lyesin, SEC13Flist Package for R, [email protected] ")
  list_all <- SEC13Flist::SEC_13F_list()
  list_all <- unique(list_all["HAS_LISTED_OPTION"])
  list_HAS_LISTED_OPTION <- nrow(subset(list_all, !HAS_LISTED_OPTION %in% c("","*")))

diff --git a/tests/testthat/test-sec13F-local.R b/tests/testthat/test-sec13F-local.R
@@ -1,4 +1,5 @@
 test_that("Local processing yields same result as via web-site", {
+ options(HTTPUserAgent="Yan Lyesin, SEC13Flist Package for R, [email protected] ")
  if (Sys.info()["sysname"] == "Windows") download.file("https://www.sec.gov/divisions/investment/13f/13flist2020q3.pdf",
  mode = "wb",
  destfile = "13flist2020q3.pdf") else download.file("https://www.sec.gov/divisions/investment/13f/13flist2020q3.pdf",

diff --git a/tests/testthat/test-status.R b/tests/testthat/test-status.R
@@ -1,4 +1,5 @@
 test_that("list of possible values for STATUS column", {
+ options(HTTPUserAgent="Yan Lyesin, SEC13Flist Package for R, [email protected] ")
  STATUS_1 <- ""
  STATUS_2 <- "ADDED"
  STATUS_3 <- "DELETED"