KiCad PCB EDA Suite
github_getliblist.cpp
Go to the documentation of this file.
1 /*
2  * This program source code file is part of KiCad, a free EDA CAD application.
3  *
4  * Copyright (C) 2015 Jean-Pierre Charras jp.charras at wanadoo.fr
5  * Copyright (C) 2013 SoftPLC Corporation, Dick Hollenbeck <dick@softplc.com>
6  * Copyright (C) 2015 KiCad Developers, see CHANGELOG.TXT for contributors.
7  *
8  * This program is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU General Public License
10  * as published by the Free Software Foundation; either version 2
11  * of the License, or (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  * GNU General Public License for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, you may find one here:
20  * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
21  * or you may search the http://www.gnu.org website for the version 2 license,
22  * or you may write to the Free Software Foundation, Inc.,
23  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
24  */
25 
26 
27 /*
28  * While creating a wizard to edit the fp lib tables, and mainly the web viewer
29  * which can read the list of pretty library on a github repos, I was told there is
30  * this URL to retrieve info from any particular repo:
31  *
32  * https://api.github.com/orgs/KiCad/repos
33  * or
34  * https://api.github.com/users/KiCad/repos
35  *
36  * This gets just information on the repo in JSON format.
37  *
38  * I used avhttp, already used in the pcbnew Github plugin to download
39  * the json file.
40  *
41  * JP Charras.
42  */
43 
44 #include <kicad_curl/kicad_curl_easy.h> // Include before any wx file
45 #include <wx/uri.h>
46 
47 #include <github_getliblist.h>
48 #include <macros.h>
49 #include <common.h>
50 #include <html_link_parser.h>
51 
52 
53 GITHUB_GETLIBLIST::GITHUB_GETLIBLIST( const wxString& aRepoURL )
54 {
55  m_repoURL = aRepoURL;
56  m_libs_ext = wxT( ".pretty" );
57  strcpy( m_option_string, "application/json" );
58 }
59 
60 
61 bool GITHUB_GETLIBLIST::Get3DshapesLibsList( wxArrayString* aList,
62  bool (*aFilter)( const wxString& aData ) )
63 {
64  std::string fullURLCommand;
65 
66  strcpy( m_option_string, "text/html" );
67 
68  wxString errorMsg;
69 
70  wxString repoURL = m_repoURL;
71  fullURLCommand = repoURL.utf8_str();
72  bool success = remoteGetJSON( fullURLCommand, &errorMsg );
73 
74  if( !success )
75  {
76  wxMessageBox( errorMsg );
77  return false;
78  }
79 
80  if( aFilter && aList )
81  {
82  //Convert m_image (std::string) to a wxString for HTML_LINK_PARSER
83  wxString buffer( GetBuffer() );
84 
85  HTML_LINK_PARSER html_parser( buffer, *aList );
86  html_parser.ParseLinks( aFilter );
87  }
88 
89  return true;
90 }
91 
92 
93 bool GITHUB_GETLIBLIST::GetFootprintLibraryList( wxArrayString& aList )
94 {
95  std::string fullURLCommand;
96  int page = 1;
97  int itemCountMax = 99; // Do not use a value >= 100, it does not work
98 
99  strcpy( m_option_string, "application/json" );
100 
101  // Github max items returned is 100 per page
102 
103  if( !repoURL2listURL( m_repoURL, &fullURLCommand, itemCountMax, page ) )
104  {
105  wxString msg = wxString::Format( _( "malformed URL:\n'%s'" ), GetChars( m_repoURL ) );
106  wxMessageBox( msg );
107  return false;
108  }
109 
110  // The URL lib names are relative to the server name.
111  // so add the server name to them.
112  wxURI repo( m_repoURL );
113  wxString urlPrefix = repo.GetScheme() + wxT( "://" ) + repo.GetServer() + wxT( "/" );
114 
115  wxString errorMsg;
116  const char sep = ','; // Separator fields, in json returned file
117  wxString tmp;
118  int items_count_per_page = 0;
119  std::string& json_image = GetBuffer();
120 
121  while( 1 )
122  {
123  bool success = remoteGetJSON( fullURLCommand, &errorMsg );
124 
125  if( !success )
126  {
127  wxMessageBox( errorMsg );
128  return false;
129  }
130 
131 
132  for( unsigned ii = 0; ii < json_image.size(); ii++ )
133  {
134  if( json_image[ii] == sep || ii == json_image.size() - 1 )
135  {
136  if( tmp.StartsWith( wxT( "\"full_name\"" ) ) )
137  {
138  #define QUOTE '\"'
139  // Remove useless quotes:
140  if( tmp[tmp.Length() - 1] == QUOTE )
141  tmp.RemoveLast();
142 
143  if( tmp.EndsWith( m_libs_ext ) )
144  {
145  aList.Add( tmp.AfterLast( ':' ) );
146  int idx = aList.GetCount() - 1;
147 
148  if( aList[idx][0] == QUOTE )
149  aList[idx].Remove( 0, 1 );
150 
151  aList[idx].Prepend( urlPrefix );
152  }
153 
154  items_count_per_page++;
155  }
156 
157  tmp.Clear();
158  }
159  else
160  tmp << json_image[ii];
161  }
162 
163  if( items_count_per_page >= itemCountMax )
164  {
165  page++;
166  repoURL2listURL( m_repoURL, &fullURLCommand, itemCountMax, page );
167  items_count_per_page = 0;
168  ClearBuffer();
169  }
170  else
171  break;
172  }
173 
174  aList.Sort();
175  return true;
176 }
177 
178 
179 bool GITHUB_GETLIBLIST::repoURL2listURL( const wxString& aRepoURL,
180  std::string* aFullURLCommand,
181  int aItemCountMax, int aPage )
182 {
183  // aListURL is e.g. "https://api.github.com/orgs/KiCad/repos"
184  // or "https://api.github.com/users/KiCad/repos"
185  // aRepoURL is e.g. "https://github.com/KiCad"
186  // Github has a default pagination set to 30 items.
187  // but allows up to 100 items max if we add the "?per_page=100" option
188 
189  wxURI repo( aRepoURL );
190 
191  if( repo.HasServer() && repo.HasPath() )
192  {
193  // goal: "https://api.github.com/users/KiCad"
194  // or "https://api.github.com/orgs/KiCad"
195  // "https://api.github.com/users/..." works both for orgs and users
196  // if we just retrieve the .pretty list
197  wxString target_url( wxT( "https://api.github.com/users" ) );
198  target_url += repo.GetPath();
199  target_url += wxT( "/repos" );
200 
201  // Github has a default pagination set to 30 items.
202  // but allows up to 100 items max. Use this limit
203  target_url += wxString::Format( "?per_page=%d&page=%d", aItemCountMax, aPage );
204 
205  *aFullURLCommand = target_url.utf8_str();
206  return true;
207  }
208 
209  return false;
210 }
211 
212 
213 bool GITHUB_GETLIBLIST::remoteGetJSON( const std::string& aFullURLCommand, wxString* aMsgError )
214 {
215  KICAD_CURL_EASY kcurl;
216 
217  wxLogDebug( wxT( "Attempting to download: " ) + aFullURLCommand );
218 
219  kcurl.SetURL( aFullURLCommand );
220  kcurl.SetUserAgent( "http://kicad-pcb.org" );
221  kcurl.SetHeader( "Accept", m_option_string );
222  kcurl.SetFollowRedirects( true );
223 
224  try
225  {
226  kcurl.Perform();
227  m_image = kcurl.GetBuffer();
228  return true;
229  }
230  catch( const IO_ERROR& ioe )
231  {
232  if( aMsgError )
233  {
234  UTF8 fmt( _( "Error fetching JSON data from URL '%s'.\nReason: '%s'" ) );
235 
236  std::string msg = StrPrintf( fmt.c_str(),
237  aFullURLCommand.c_str(),
238  TO_UTF8( ioe.What() ) );
239 
240  *aMsgError = FROM_UTF8( msg.c_str() );
241  }
242  return false;
243  }
244 }
Class UTF8 is an 8 bit string that is assuredly encoded in UTF8, and supplies special conversion supp...
Definition: utf8.h:73
bool SetUserAgent(const std::string &aAgent)
Function SetUserAgent sets the request user agent.
void Perform()
Function perform equivalent to curl_easy_perform.
bool SetFollowRedirects(bool aFollow)
Function SetFollowRedirects enables the following of HTTP(s) and other redirects, by default curl doe...
GITHUB_GETLIBLIST(const wxString &aRepoURL)
static wxString FROM_UTF8(const char *cstring)
function FROM_UTF8 converts a UTF8 encoded C string to a wxString for all wxWidgets build modes...
Definition: macros.h:53
Class KICAD_CURL_EASY wrapper interface around the curl_easy API.
char m_option_string[64]
option for transfert type, like "application/json"
bool GetFootprintLibraryList(wxArrayString &aList)
Fills aList by the name of footprint libraries found on the github repo.
This file contains miscellaneous commonly used macros and functions.
int StrPrintf(std::string *result, const char *format,...)
Function StrPrintf is like sprintf() but the output is appended to a std::string instead of to a char...
Definition: richio.cpp:74
#define TO_UTF8(wxstring)
Macro TO_UTF8 converts a wxString to a UTF8 encoded C string for all wxWidgets build modes...
Definition: macros.h:47
const std::string & GetBuffer()
Function GetBuffer returns a const reference to the recevied data buffer.
bool Get3DshapesLibsList(wxArrayString *aList, bool(*aFilter)(const wxString &aData))
Fills aList by the URL of libraries found on the github repo.
std::string m_image
image of the downloaded data in its entirety.
bool SetURL(const std::string &aURL)
Function SetURL sets the request URL.
bool repoURL2listURL(const wxString &aRepoURL, std::string *aFullURLCommand, int aItemCountMax, int aPage=1)
Function repoURL2listURL translates a repo URL to the URL name which gives the state of repos URL as ...
#define QUOTE
virtual const wxString What() const
A composite of Problem() and Where()
Definition: exceptions.cpp:33
static const wxChar * GetChars(const wxString &s)
Function GetChars returns a wxChar* to the actual wxChar* data within a wxString, and is helpful for ...
Definition: macros.h:92
void Format(OUTPUTFORMATTER *out, int aNestLevel, int aCtl, CPTREE &aTree)
Function Format outputs a PTREE into s-expression format via an OUTPUTFORMATTER derivative.
Definition: ptree.cpp:205
bool remoteGetJSON(const std::string &aFullURLCommand, wxString *aMsgError)
Function remoteGetJSON Download a json text from a github repo.
const char * c_str() const
Definition: utf8.h:107
wxString m_repoURL
the URL of the Github repo
wxString m_libs_ext
the extension of the name of libraries (default = .pretty)
The common library.
void ClearBuffer()
Clear the buffer which stores all the downloaded raw data.
void SetHeader(const std::string &aName, const std::string &aValue)
Function SetHeader sets an arbitrary header for the HTTP(s) request.
Struct IO_ERROR is a class used to hold an error message and may be used when throwing exceptions con...
Definition: ki_exception.h:47
std::string & GetBuffer()